From c404b630a025a1bd128563c2fb6919b88f78d1d0 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia <35884424+meshwa19@users.noreply.github.com> Date: Mon, 23 Jan 2023 12:43:22 -0800 Subject: [PATCH] High Resolution Metrics Support (#129) --- README.md | 15 +++++- .../src/main/java/emf/canary/ECSRunnable.java | 3 +- examples/agent/src/main/java/agent/App.java | 2 + examples/ecs-firelens/src/main/java/App.java | 2 + examples/lambda/src/main/java/Handler.java | 2 + .../emf/logger/MetricsLogger.java | 51 +++++++++++++++++-- .../emf/model/MetricDefinition.java | 28 ++++++++-- .../emf/model/MetricDirective.java | 15 +++++- .../emf/model/MetricsContext.java | 46 +++++++++++++++-- .../emf/model/StorageResolution.java | 33 ++++++++++++ .../serializers/StorageResolutionFilter.java | 15 ++++++ .../StorageResolutionSerializer.java | 40 +++++++++++++++ .../cloudwatchlogs/emf/util/Validator.java | 26 +++++++++- .../emf/logger/MetricsLoggerTest.java | 30 ++++++++++- .../emf/model/MetricDefinitionTest.java | 39 +++++++++++++- .../emf/model/MetricDirectiveTest.java | 45 ++++++++++++++++ .../emf/model/MetricsContextTest.java | 4 +- 17 files changed, 376 insertions(+), 20 deletions(-) create mode 100644 src/main/java/software/amazon/cloudwatchlogs/emf/model/StorageResolution.java create mode 100644 src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionFilter.java create mode 100644 src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionSerializer.java diff --git a/README.md b/README.md index f49a6dd2..adaa78b8 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ class Example { try { metrics.putDimensions(DimensionSet.of("Service", "Aggregator")); - metrics.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS); + metrics.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS, StorageResolution.STANDARD); + metrics.putMetric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH); } catch (InvalidDimensionException | InvalidMetricException e) { log.error(e); } @@ -87,10 +88,12 @@ environment.getSink().shutdown().orTimeout(10_000L, TimeUnit.MILLISECONDS); The `MetricsLogger` is the interface you will use to publish embedded metrics. +- MetricsLogger **putMetric**(String key, double value, Unit unit, StorageResolution storageResolution) +- MetricsLogger **putMetric**(String key, double value, StorageResolution storageResolution) - MetricsLogger **putMetric**(String key, double value, Unit unit) - MetricsLogger **putMetric**(String key, double value) -Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. The Embedded Metric Format supports a maximum of 100 values per key. +Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. Multiple metrics cannot have the same key and different storage resolutions. The Embedded Metric Format supports a maximum of 100 values per key. Requirements: @@ -99,10 +102,18 @@ Requirements: - Values must be in the range of 8.515920e-109 to 1.174271e+108. In addition, special values (for example, NaN, +Infinity, -Infinity) are not supported. - Metrics must meet CloudWatch Metrics requirements, otherwise a `InvalidMetricException` will be thrown. See [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) for valid values. +- ##### Storage Resolution +An OPTIONAL value representing the storage resolution for the corresponding metric. Setting this to `High` specifies this metric as a high-resolution metric, so that CloudWatch stores the metric with sub-minute resolution down to one second. Setting this to `Standard` specifies this metric as a standard-resolution metric, which CloudWatch stores at 1-minute resolution. If a value is not provided, then a default value of `Standard` is assumed. See [Cloud Watch High-Resolution metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics) + Examples: ```java +// Standard Resolution example putMetric("Latency", 200, Unit.MILLISECONDS) +putMetric("Latency", 201, Unit.MILLISECONDS, StorageResolution.STANDARD) + +// High Resolution example +putMetric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH); ``` - MetricsLogger **putProperty**(String key, Object value ) diff --git a/canarytests/agent/src/main/java/emf/canary/ECSRunnable.java b/canarytests/agent/src/main/java/emf/canary/ECSRunnable.java index 65cfc328..d0e2a8bd 100644 --- a/canarytests/agent/src/main/java/emf/canary/ECSRunnable.java +++ b/canarytests/agent/src/main/java/emf/canary/ECSRunnable.java @@ -8,6 +8,7 @@ import software.amazon.cloudwatchlogs.emf.exception.InvalidNamespaceException; import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; import software.amazon.cloudwatchlogs.emf.model.DimensionSet; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import software.amazon.cloudwatchlogs.emf.model.Unit; import java.lang.management.ManagementFactory; @@ -46,7 +47,7 @@ public void run() { try { logger.putMetric("Invoke", 1, Unit.COUNT); logger.putMetric("Memory.HeapTotal", heapTotal, Unit.COUNT); - logger.putMetric("Memory.HeapUsed", heapUsed, Unit.COUNT); + logger.putMetric("Memory.HeapUsed", heapUsed, Unit.COUNT, StorageResolution.HIGH); logger.putMetric("Memory.JVMUsedTotal", heapUsed + nonHeapUsed, Unit.COUNT); } catch (InvalidMetricException e) { System.out.println(e); diff --git a/examples/agent/src/main/java/agent/App.java b/examples/agent/src/main/java/agent/App.java index 163200be..3a300aa6 100644 --- a/examples/agent/src/main/java/agent/App.java +++ b/examples/agent/src/main/java/agent/App.java @@ -9,6 +9,7 @@ import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; import software.amazon.cloudwatchlogs.emf.model.DimensionSet; import software.amazon.cloudwatchlogs.emf.model.Unit; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import java.util.concurrent.TimeUnit; @@ -35,6 +36,7 @@ private static void emitMetric(Environment environment) MetricsLogger logger = new MetricsLogger(environment); logger.setDimensions(DimensionSet.of("Operation", "Agent")); logger.putMetric("ExampleMetric", 100, Unit.MILLISECONDS); + logger.putMetric("ExampleHighResolutionMetric", 10, Unit.MILLISECONDS, StorageResolution.HIGH); logger.putProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8"); logger.flush(); } diff --git a/examples/ecs-firelens/src/main/java/App.java b/examples/ecs-firelens/src/main/java/App.java index 344aa003..0acc6797 100644 --- a/examples/ecs-firelens/src/main/java/App.java +++ b/examples/ecs-firelens/src/main/java/App.java @@ -23,6 +23,7 @@ import software.amazon.cloudwatchlogs.emf.environment.EnvironmentProvider; import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException; import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import software.amazon.cloudwatchlogs.emf.model.Unit; import sun.misc.Signal; import java.io.IOException; @@ -39,6 +40,7 @@ public static void main(String[] args) throws Exception { MetricsLogger logger = new MetricsLogger(); logger.setNamespace("FargateEMF"); logger.putMetric("Latency", 63, Unit.MILLISECONDS); + logger.putMetric("CPU Utilization", 87, Unit.PERCENT, StorageResolution.HIGH); logger.flush(); HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0); int portNumber = 8000; diff --git a/examples/lambda/src/main/java/Handler.java b/examples/lambda/src/main/java/Handler.java index 6feb3e5f..4a4e1f05 100644 --- a/examples/lambda/src/main/java/Handler.java +++ b/examples/lambda/src/main/java/Handler.java @@ -5,6 +5,7 @@ import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException; import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; import software.amazon.cloudwatchlogs.emf.model.DimensionSet; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import software.amazon.cloudwatchlogs.emf.model.Unit; import java.util.HashMap; @@ -20,6 +21,7 @@ public String handleRequest(Map event, Context context) { try { logger.putDimensions(DimensionSet.of("Service", "Aggregator")); logger.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS); + logger.putMetric("CPU Utilization", 87, Unit.PERCENT, StorageResolution.HIGH); } catch (InvalidDimensionException | InvalidMetricException | DimensionSetExceededException e) { System.out.println(e); } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLogger.java b/src/main/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLogger.java index cf4a0cb6..00c955f7 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLogger.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLogger.java @@ -32,6 +32,7 @@ import software.amazon.cloudwatchlogs.emf.exception.InvalidTimestampException; import software.amazon.cloudwatchlogs.emf.model.DimensionSet; import software.amazon.cloudwatchlogs.emf.model.MetricsContext; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import software.amazon.cloudwatchlogs.emf.model.Unit; import software.amazon.cloudwatchlogs.emf.sinks.ISink; @@ -98,7 +99,7 @@ public void flush() { } /** - * Set a property on the published metrics. This is stored in the emitted log data and you are + * Set a property on the published metrics. This is stored in the emitted log data, and you are * not charged for this data by CloudWatch Metrics. These values can be values that are useful * for searching on, but have too high cardinality to emit as dimensions to CloudWatch Metrics. * @@ -189,20 +190,62 @@ public MetricsLogger resetDimensions(boolean useDefault) { * @param key is the name of the metric * @param value is the value of the metric * @param unit is the unit of the metric value + * @param storageResolution is the resolution of the metric + * @see CloudWatch + * High Resolution Metrics * @return the current logger * @throws InvalidMetricException if the metric is invalid */ - public MetricsLogger putMetric(String key, double value, Unit unit) + public MetricsLogger putMetric( + String key, double value, Unit unit, StorageResolution storageResolution) throws InvalidMetricException { rwl.readLock().lock(); try { - this.context.putMetric(key, value, unit); + this.context.putMetric(key, value, unit, storageResolution); return this; } finally { rwl.readLock().unlock(); } } + /** + * Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does + * not contribute to your account TPS limits. The value will also be available in your + * CloudWatch Logs + * + * @param key is the name of the metric + * @param value is the value of the metric + * @param storageResolution is the resolution of the metric + * @see CloudWatch + * High Resolution Metrics + * @return the current logger + * @throws InvalidMetricException if the metric is invalid + */ + public MetricsLogger putMetric(String key, double value, StorageResolution storageResolution) + throws InvalidMetricException { + this.putMetric(key, value, Unit.NONE, storageResolution); + return this; + } + + /** + * Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does + * not contribute to your account TPS limits. The value will also be available in your + * CloudWatch Logs + * + * @param key is the name of the metric + * @param value is the value of the metric + * @param unit is the unit of the metric value + * @return the current logger + * @throws InvalidMetricException if the metric is invalid + */ + public MetricsLogger putMetric(String key, double value, Unit unit) + throws InvalidMetricException { + this.putMetric(key, value, unit, StorageResolution.STANDARD); + return this; + } + /** * Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does * not contribute to your account TPS limits. The value will also be available in your @@ -214,7 +257,7 @@ public MetricsLogger putMetric(String key, double value, Unit unit) * @throws InvalidMetricException if the metric is invalid */ public MetricsLogger putMetric(String key, double value) throws InvalidMetricException { - this.putMetric(key, value, Unit.NONE); + this.putMetric(key, value, Unit.NONE, StorageResolution.STANDARD); return this; } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java index be9b9150..957da0d3 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java @@ -17,6 +17,7 @@ package software.amazon.cloudwatchlogs.emf.model; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -26,6 +27,9 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; +import lombok.Setter; +import software.amazon.cloudwatchlogs.emf.serializers.StorageResolutionFilter; +import software.amazon.cloudwatchlogs.emf.serializers.StorageResolutionSerializer; import software.amazon.cloudwatchlogs.emf.serializers.UnitDeserializer; import software.amazon.cloudwatchlogs.emf.serializers.UnitSerializer; @@ -43,18 +47,36 @@ class MetricDefinition { @JsonDeserialize(using = UnitDeserializer.class) private Unit unit; + @Getter + @Setter + @JsonProperty("StorageResolution") + @JsonInclude( + value = JsonInclude.Include.CUSTOM, + valueFilter = + StorageResolutionFilter.class) // Do not serialize when valueFilter is true + @JsonSerialize(using = StorageResolutionSerializer.class) + public StorageResolution storageResolution; + @JsonIgnore @NonNull @Getter private List values; MetricDefinition(String name) { - this(name, Unit.NONE, new ArrayList<>()); + this(name, Unit.NONE, StorageResolution.STANDARD, new ArrayList<>()); } MetricDefinition(String name, double value) { - this(name, Unit.NONE, value); + this(name, Unit.NONE, StorageResolution.STANDARD, value); } MetricDefinition(String name, Unit unit, double value) { - this(name, unit, new ArrayList<>(Arrays.asList(value))); + this(name, unit, StorageResolution.STANDARD, new ArrayList<>(Arrays.asList(value))); + } + + MetricDefinition(String name, StorageResolution storageResolution, double value) { + this(name, Unit.NONE, storageResolution, new ArrayList<>(Arrays.asList(value))); + } + + MetricDefinition(String name, Unit unit, StorageResolution storageResolution, double value) { + this(name, unit, storageResolution, new ArrayList<>(Arrays.asList(value))); } void addValue(double value) { diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java index 0846b2c4..80ebb95a 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java @@ -66,15 +66,26 @@ void putDimensionSet(DimensionSet dimensionSet) { dimensions.add(dimensionSet); } + // Helper method for testing putMetric() void putMetric(String key, double value) { - putMetric(key, value, Unit.NONE); + putMetric(key, value, Unit.NONE, StorageResolution.STANDARD); } + // Helper method for testing putMetric() void putMetric(String key, double value, Unit unit) { + putMetric(key, value, unit, StorageResolution.STANDARD); + } + + // Helper method for testing serialization + void putMetric(String key, double value, StorageResolution storageResolution) { + putMetric(key, value, Unit.NONE, storageResolution); + } + + void putMetric(String key, double value, Unit unit, StorageResolution storageResolution) { metrics.compute( key, (k, v) -> { - if (v == null) return new MetricDefinition(key, unit, value); + if (v == null) return new MetricDefinition(key, unit, storageResolution, value); else { v.addValue(value); return v; diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java index 7db829f7..14fae1b6 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java @@ -34,6 +34,7 @@ public class MetricsContext { @Getter private final RootNode rootNode; private MetricDirective metricDirective; + private final Map metricNameAndResolutionMap = new HashMap<>(); public MetricsContext() { this(new RootNode()); @@ -108,6 +109,44 @@ public boolean hasDefaultDimensions() { * an array of scalar values. * * 
{@code
+     * metricContext.putMetric("Latency", 100, Unit.MILLISECONDS, StorageResolution.HIGH)
+     * }
+ * + * @param key Name of the metric + * @param value Value of the metric + * @param unit The unit of the metric + * @param storageResolution The resolution of the metric + * @throws InvalidMetricException if the metric is invalid + */ + public void putMetric(String key, double value, Unit unit, StorageResolution storageResolution) + throws InvalidMetricException { + Validator.validateMetric(key, value, unit, storageResolution, metricNameAndResolutionMap); + metricDirective.putMetric(key, value, unit, storageResolution); + metricNameAndResolutionMap.put(key, storageResolution); + } + /** + * Add a metric measurement to the context with a storage resolution but without a unit. + * Multiple calls using the same key will be stored as an array of scalar values. + * + * 
{@code
+     * metricContext.putMetric("Latency", 100, StorageResolution.HIGH)
+     * }
+ * + * @param key Name of the metric + * @param value Value of the metric + * @param storageResolution The resolution of the metric + * @throws InvalidMetricException if the metric is invalid + */ + public void putMetric(String key, double value, StorageResolution storageResolution) + throws InvalidMetricException { + putMetric(key, value, Unit.NONE, storageResolution); + } + + /** + * Add a metric measurement to the context without a storage resolution. Multiple calls using + * the same key will be stored as an array of scalar values. + * + * 
{@code
      * metricContext.putMetric("Latency", 100, Unit.MILLISECONDS)
      * }
* @@ -117,8 +156,7 @@ public boolean hasDefaultDimensions() { * @throws InvalidMetricException if the metric is invalid */ public void putMetric(String key, double value, Unit unit) throws InvalidMetricException { - Validator.validateMetric(key, value, unit); - metricDirective.putMetric(key, value, unit); + putMetric(key, value, unit, StorageResolution.STANDARD); } /** @@ -134,7 +172,7 @@ public void putMetric(String key, double value, Unit unit) throws InvalidMetricE * @throws InvalidMetricException if the metric is invalid */ public void putMetric(String key, double value) throws InvalidMetricException { - putMetric(key, value, Unit.NONE); + putMetric(key, value, Unit.NONE, StorageResolution.STANDARD); } /** @@ -297,12 +335,14 @@ public List serialize() throws JsonProcessingException { new MetricDefinition( metric.getName(), metric.getUnit(), + metric.getStorageResolution(), metric.getValues() .subList(0, Constants.MAX_DATAPOINTS_PER_METRIC))); metricDefinitions.offer( new MetricDefinition( metric.getName(), metric.getUnit(), + metric.getStorageResolution(), metric.getValues() .subList( Constants.MAX_DATAPOINTS_PER_METRIC, diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/StorageResolution.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/StorageResolution.java new file mode 100644 index 00000000..cd69eb73 --- /dev/null +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/StorageResolution.java @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.cloudwatchlogs.emf.model; + +public enum StorageResolution { + STANDARD(60), + HIGH(1), + UNKNOWN_TO_SDK_VERSION(-1); + + private final int value; + + StorageResolution(final int newValue) { + value = newValue; + } + + public int getValue() { + return this.value; + } +} diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionFilter.java b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionFilter.java new file mode 100644 index 00000000..ec0d1c9f --- /dev/null +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionFilter.java @@ -0,0 +1,15 @@ +package software.amazon.cloudwatchlogs.emf.serializers; + +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; + +public class StorageResolutionFilter { + + @Override + public boolean equals(Object obj) { + + if (!(obj instanceof StorageResolution)) { + return false; + } + return (obj.toString().equals("STANDARD")); + } +} diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionSerializer.java b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionSerializer.java new file mode 100644 index 00000000..ab5d9a87 --- /dev/null +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/StorageResolutionSerializer.java @@ -0,0 +1,40 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.cloudwatchlogs.emf.serializers; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; + +public class StorageResolutionSerializer extends StdSerializer { + StorageResolutionSerializer() { + this(null); + } + + StorageResolutionSerializer(Class t) { + super(t); + } + + @Override + public void serialize(StorageResolution value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + int resolution = value.getValue(); + jgen.writeNumber(resolution); + } +} diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/util/Validator.java b/src/main/java/software/amazon/cloudwatchlogs/emf/util/Validator.java index 18a8981f..d8bf54b4 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/util/Validator.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/util/Validator.java @@ -17,12 +17,15 @@ package software.amazon.cloudwatchlogs.emf.util; import java.time.Instant; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import software.amazon.cloudwatchlogs.emf.Constants; import software.amazon.cloudwatchlogs.emf.exception.*; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; import software.amazon.cloudwatchlogs.emf.model.Unit; public class Validator { + private Validator() { throw new IllegalStateException("Utility class"); } @@ -88,10 +91,18 @@ public static void validateDimensionSet(String dimensionName, String dimensionVa * @param name Metric name * @param value Metric value * @param unit Metric unit + * @param storageResolution Metric resolution + * @param metricNameAndResolutionMap Map to validate Metric * @throws InvalidMetricException if metric is invalid */ - public static void validateMetric(String name, double value, Unit unit) + public static void validateMetric( + String name, + double value, + Unit unit, + StorageResolution storageResolution, + Map metricNameAndResolutionMap) throws InvalidMetricException { + if (name == null || name.trim().isEmpty()) { throw new InvalidMetricException( "Metric name " + name + " must include at least one non-whitespace character"); @@ -112,6 +123,19 @@ public static void validateMetric(String name, double value, Unit unit) if (unit == null) { throw new InvalidMetricException("Metric unit cannot be null"); } + + if (storageResolution == null + || storageResolution == StorageResolution.UNKNOWN_TO_SDK_VERSION) { + throw new InvalidMetricException("Metric resolution is invalid"); + } + + if ((metricNameAndResolutionMap.containsKey(name)) + && (!metricNameAndResolutionMap.get(name).equals(storageResolution))) { + throw new InvalidMetricException( + "Resolution for metric " + + name + + " is already set. A single log event cannot have a metric with two different resolutions."); + } } /** diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLoggerTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLoggerTest.java index 0539ccc1..683fa75f 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLoggerTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLoggerTest.java @@ -38,6 +38,8 @@ import software.amazon.cloudwatchlogs.emf.exception.InvalidTimestampException; import software.amazon.cloudwatchlogs.emf.model.DimensionSet; import software.amazon.cloudwatchlogs.emf.model.MetricsContext; +import software.amazon.cloudwatchlogs.emf.model.StorageResolution; +import software.amazon.cloudwatchlogs.emf.model.Unit; import software.amazon.cloudwatchlogs.emf.sinks.SinkShunt; class MetricsLoggerTest { @@ -349,7 +351,33 @@ void whenPutMetric_withInvalidValue_thenThrowInvalidMetricException(double value @Test void whenPutMetric_withNullUnit_thenThrowInvalidMetricException() { - assertThrows(InvalidMetricException.class, () -> logger.putMetric("test", 1, null)); + assertThrows(InvalidMetricException.class, () -> logger.putMetric("test", 1, (Unit) null)); + } + + @Test + void whenPutMetric_withNullStorageResolution_thenThrowInvalidMetricException() { + assertThrows( + InvalidMetricException.class, + () -> logger.putMetric("test", 1, (StorageResolution) null)); + } + + @Test + void whenPutMetric_withSameMetricDifferentStorageResolutionAfterFlush_thenAllow() + throws InvalidMetricException { + logger.putMetric("Count", 1); + logger.flush(); + logger.putMetric("Count", 1.0, StorageResolution.HIGH); + logger.flush(); + assertTrue(sink.getLogEvents().get(0).contains("StorageResolution")); + } + + @Test + void whenPutMetric_withDifferentStorageResolution_thenThrowInvalidMetricException() + throws InvalidMetricException { + logger.putMetric("test", 1); + assertThrows( + InvalidMetricException.class, + () -> logger.putMetric("test", 1, StorageResolution.HIGH)); } @Test diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java index cc1aeaff..25c3d43a 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java @@ -31,6 +31,38 @@ public void testThrowExceptionIfNameIsNull() { new MetricDefinition(null); } + @Test + public void testSerializeMetricDefinitionWithoutUnitWithHighStorageResolution() + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + MetricDefinition metricDefinition = + new MetricDefinition("Time", StorageResolution.HIGH, 10); + String metricString = objectMapper.writeValueAsString(metricDefinition); + + assertEquals("{\"Name\":\"Time\",\"Unit\":\"None\",\"StorageResolution\":1}", metricString); + } + + @Test + public void testSerializeMetricDefinitionWithUnitWithoutStorageResolution() + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS, 10); + String metricString = objectMapper.writeValueAsString(metricDefinition); + + assertEquals("{\"Name\":\"Time\",\"Unit\":\"Milliseconds\"}", metricString); + } + + @Test + public void testSerializeMetricDefinitionWithoutUnitWithStandardStorageResolution() + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + MetricDefinition metricDefinition = + new MetricDefinition("Time", StorageResolution.STANDARD, 10); + String metricString = objectMapper.writeValueAsString(metricDefinition); + + assertEquals("{\"Name\":\"Time\",\"Unit\":\"None\"}", metricString); + } + @Test public void testSerializeMetricDefinitionWithoutUnit() throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); @@ -43,10 +75,13 @@ public void testSerializeMetricDefinitionWithoutUnit() throws JsonProcessingExce @Test public void testSerializeMetricDefinition() throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); - MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS, 10); + MetricDefinition metricDefinition = + new MetricDefinition("Time", Unit.MILLISECONDS, StorageResolution.HIGH, 10); String metricString = objectMapper.writeValueAsString(metricDefinition); - assertEquals("{\"Name\":\"Time\",\"Unit\":\"Milliseconds\"}", metricString); + assertEquals( + "{\"Name\":\"Time\",\"Unit\":\"Milliseconds\",\"StorageResolution\":1}", + metricString); } @Test diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java index 3a3385f1..9c095c50 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java @@ -90,6 +90,51 @@ void testPutMetricWithUnit() { Unit.MILLISECONDS, metricDirective.getMetrics().get("Time").getUnit()); } + @Test + void testPutMetricWithoutStorageResolution() throws JsonProcessingException { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10); + + String serializedMetricDirective = objectMapper.writeValueAsString(metricDirective); + + Assertions.assertEquals( + StorageResolution.STANDARD, + metricDirective.getMetrics().get("Time").getStorageResolution()); + Assertions.assertEquals( + "{\"Dimensions\":[[]],\"Metrics\":[{\"Name\":\"Time\",\"Unit\":\"None\"}],\"Namespace\":\"aws-embedded-metrics\"}", + serializedMetricDirective); + } + + @Test + void testPutMetricWithStandardStorageResolution() throws JsonProcessingException { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10, StorageResolution.STANDARD); + + String serializedMetricDirective = objectMapper.writeValueAsString(metricDirective); + + Assertions.assertEquals( + StorageResolution.STANDARD, + metricDirective.getMetrics().get("Time").getStorageResolution()); + Assertions.assertEquals( + "{\"Dimensions\":[[]],\"Metrics\":[{\"Name\":\"Time\",\"Unit\":\"None\"}],\"Namespace\":\"aws-embedded-metrics\"}", + serializedMetricDirective); + } + + @Test + void testPutMetricWithHighStorageResolution() throws JsonProcessingException { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10, StorageResolution.HIGH); + + String serializedMetricDirective = objectMapper.writeValueAsString(metricDirective); + + Assertions.assertEquals( + StorageResolution.HIGH, + metricDirective.getMetrics().get("Time").getStorageResolution()); + Assertions.assertEquals( + "{\"Dimensions\":[[]],\"Metrics\":[{\"Name\":\"Time\",\"StorageResolution\":1,\"Unit\":\"None\"}],\"Namespace\":\"aws-embedded-metrics\"}", + serializedMetricDirective); + } + @Test void testPutDimensions() throws JsonProcessingException, InvalidDimensionException, diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java index 3a39baaa..4cffe824 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java @@ -201,7 +201,9 @@ private ArrayList parseMetrics(String event) throws JsonProces Unit unit = Unit.fromValue(metric.get("Unit")); Object value = rootNode.get(name); if (value instanceof ArrayList) { - metricDefinitions.add(new MetricDefinition(name, unit, (ArrayList) value)); + metricDefinitions.add( + new MetricDefinition( + name, unit, StorageResolution.STANDARD, (ArrayList) value)); } else { metricDefinitions.add(new MetricDefinition(name, unit, (double) value)); }