Skip to content

Commit

Permalink
Revert "Revert "Gjoranv/add default public metrics consumer" MERGEOK"
Browse files Browse the repository at this point in the history
  • Loading branch information
gjoranv authored Jun 25, 2019
1 parent bfc319f commit 30e8e13
Show file tree
Hide file tree
Showing 17 changed files with 344 additions and 98 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ private static Metric combineMetrics(@Nullable Metric original, Metric newMetric
return original != null ? newMetric.addDimensionsFrom(original) : newMetric;
}

private static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) {
static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) {
Consumer.Builder builder = new Consumer.Builder().name(consumer.getId());
consumer.getMetrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric)));
return builder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@

import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics;
import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers;
import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION_ID;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.INSTANCE;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE;
import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.getDefaultPublicConsumer;
import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer;
import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet;
import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS;
Expand Down Expand Up @@ -128,8 +130,10 @@ public void getConfig(MonitoringConfig.Builder builder) {

@Override
public void getConfig(ConsumersConfig.Builder builder) {
var amendedDefaultConsumer = addMetrics(getVespaMetricsConsumer(), getAdditionalDefaultMetrics().getMetrics());
builder.consumer.addAll(generateConsumers(amendedDefaultConsumer, getUserMetricsConsumers()));
var amendedVespaConsumer = addMetrics(getVespaMetricsConsumer(), getAdditionalDefaultMetrics().getMetrics());
builder.consumer.addAll(generateConsumers(amendedVespaConsumer, getUserMetricsConsumers()));

if (! isHostedVespa()) builder.consumer.add(toConsumerBuilder(getDefaultPublicConsumer()));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
*/

package com.yahoo.vespa.model.admin.monitoring;

import ai.vespa.metricsproxy.http.GenericMetricsHandler;
import com.google.common.collect.ImmutableList;

import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
import static java.util.Collections.emptyList;

/**
* @author gjoranv
*/
public class DefaultPublicConsumer {

public static final String DEFAULT_PUBLIC_CONSUMER_ID = GenericMetricsHandler.DEFAULT_PUBLIC_CONSUMER_ID.id;

private static final MetricSet publicConsumerMetrics = new MetricSet("public-consumer-metrics",
emptyList(),
ImmutableList.of(defaultPublicMetricSet,
defaultVespaMetricSet,
systemMetricSet));

public static MetricsConsumer getDefaultPublicConsumer() {
return new MetricsConsumer(DEFAULT_PUBLIC_CONSUMER_ID, publicConsumerMetrics);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
*/

package com.yahoo.vespa.model.admin.monitoring;


import com.google.common.collect.ImmutableSet;

import java.util.LinkedHashSet;
import java.util.Set;

import static java.util.Collections.emptyList;

/**
* TODO: Add content metrics.
*
* @author gjoranv
*/
public class DefaultPublicMetrics {

public static MetricSet defaultPublicMetricSet = createMetricSet();

private static MetricSet createMetricSet() {
return new MetricSet("public",
getAllMetrics(),
emptyList());
}

private static Set<Metric> getAllMetrics() {
return ImmutableSet.<Metric>builder()
.addAll(getContainerMetrics())
.addAll(getQrserverMetrics())
.build();
}

private static Set<Metric> getContainerMetrics() {
Set<Metric> metrics = new LinkedHashSet<>();

metrics.add(new Metric("http.status.1xx.rate"));
metrics.add(new Metric("http.status.2xx.rate"));
metrics.add(new Metric("http.status.3xx.rate"));
metrics.add(new Metric("http.status.4xx.rate"));
metrics.add(new Metric("http.status.5xx.rate"));
metrics.add(new Metric("jdisc.gc.ms.average"));
metrics.add(new Metric("mem.heap.free.average"));

return metrics;
}

private static Set<Metric> getQrserverMetrics() {
Set<Metric> metrics = new LinkedHashSet<>();

metrics.add(new Metric("queries.rate"));
metrics.add(new Metric("query_latency.average"));
metrics.add(new Metric("query_latency.95percentile"));
metrics.add(new Metric("query_latency.99percentile"));
metrics.add(new Metric("hits_per_query.average"));
metrics.add(new Metric("totalhits_per_query.average"));
metrics.add(new Metric("degraded_queries.rate"));
metrics.add(new Metric("failed_queries.rate"));
metrics.add(new Metric("serverActiveThreads.average"));

return metrics;
}

private DefaultPublicMetrics() { }

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
/**
* @author gjoranv
*/
@SuppressWarnings("UnusedDeclaration") // Used by model amenders
public class SystemMetrics {
public static final String CPU_UTIL = "cpu.util";
public static final String CPU_SYS_UTIL = "cpu.sys.util";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*
* @author gjoranv
*/
@SuppressWarnings("UnusedDeclaration") // Used by model amenders
public class VespaMetricSet {

public static final MetricSet vespaMetricSet = new MetricSet("vespa",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import static java.util.Collections.emptyList;

/**
* This class sets up the 'Vespa' metrics consumer.
* This class sets up the 'Vespa' metrics consumer, which is mainly used for Yamas in hosted Vespa.
*
* @author trygve
* @author gjoranv
Expand All @@ -19,14 +19,14 @@ public class VespaMetricsConsumer {

public static final String VESPA_CONSUMER_ID = VespaMetrics.VESPA_CONSUMER_ID.id;

private static final MetricSet defaultConsumerMetrics = new MetricSet("vespa-consumer-metrics",
emptyList(),
ImmutableList.of(vespaMetricSet,
systemMetricSet,
networkMetricSet));
private static final MetricSet vespaConsumerMetrics = new MetricSet("vespa-consumer-metrics",
emptyList(),
ImmutableList.of(vespaMetricSet,
systemMetricSet,
networkMetricSet));

public static MetricsConsumer getVespaMetricsConsumer() {
return new MetricsConsumer(VESPA_CONSUMER_ID, defaultConsumerMetrics);
return new MetricsConsumer(VESPA_CONSUMER_ID, vespaConsumerMetrics);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.util.LinkedHashMap;
import java.util.Map;

import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
Expand All @@ -20,6 +21,7 @@
public class PredefinedMetricSets {

public static final Map<String, MetricSet> predefinedMetricSets = toMapById(
defaultPublicMetricSet,
defaultVespaMetricSet,
vespaMetricSet,
systemMetricSet,
Expand All @@ -28,8 +30,11 @@ public class PredefinedMetricSets {

private static Map<String, MetricSet> toMapById(MetricSet... metricSets) {
Map<String, MetricSet> availableMetricSets = new LinkedHashMap<>();
for (MetricSet metricSet : metricSets)
availableMetricSets.put(metricSet.getId(), metricSet);
for (MetricSet metricSet : metricSets) {
var existing = availableMetricSets.put(metricSet.getId(), metricSet);
if (existing != null)
throw new IllegalArgumentException("There are two predefined metric sets with id " + existing.getId());
}
return Collections.unmodifiableMap(availableMetricSets);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.Map;
import java.util.stream.Collectors;

import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
Expand Down Expand Up @@ -77,6 +78,10 @@ private static String metricSetId(String consumerName) {
private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) {
if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE)
throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)");

if (consumerId.equalsIgnoreCase(DEFAULT_PUBLIC_CONSUMER_ID))
throw new IllegalArgumentException("'" + DEFAULT_PUBLIC_CONSUMER_ID + "' is not allowed as metrics consumer id (case is ignored.)");

if (metrics.hasConsumerIgnoreCase(consumerId))
throw new IllegalArgumentException("'" + consumerId + "' is used as id for two metrics consumers (case is ignored.)");
}
Expand Down
Loading

0 comments on commit 30e8e13

Please sign in to comment.