Merge pull request #545 from solarwinds/bugfix/NH-67458-fix-istio-dur…

…ation-metric fixing istio duration metric calculation
solarwinds · Jan 23, 2024 · e054d3a · e054d3a
2 parents 9ec54f3 + 61b0941
commit e054d3a
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 21 deletions.
diff --git a/deploy/helm/CHANGELOG.md b/deploy/helm/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [3.2.0-alpha.17] - 2024-01-23
+
+### Fixed
+
+- Fixed how `k8s.istio_request_duration_milliseconds.rate` metric is calculated
+
 ## [3.2.0-alpha.16] - 2024-01-22
 
 ### Fixed

diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: swo-k8s-collector
-version: 3.2.0-alpha.16
+version: 3.2.0-alpha.17
 appVersion: "0.9.2"
 description: SolarWinds Kubernetes Integration
 keywords:

diff --git a/deploy/helm/node-collector-config.yaml b/deploy/helm/node-collector-config.yaml
@@ -294,12 +294,14 @@ processors:
       - context: metric
         statements:
           - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+          - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
           - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
           - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-          - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
           - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
           - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
           - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+          - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name == "k8s.istio_request_duration_milliseconds_sum"
+          - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name == "k8s.istio_request_duration_milliseconds_count"
 
   metricstransform/istio-metrics:
     transforms:
@@ -324,7 +326,8 @@ processors:
       metrics:
         - k8s.istio_request_bytes.rate
         - k8s.istio_response_bytes.rate
-        - k8s.istio_request_duration_milliseconds.rate
+        - k8s.istio_request_duration_milliseconds_sum_temp
+        - k8s.istio_request_duration_milliseconds_count_temp
         - k8s.istio_requests.rate
         - k8s.istio_tcp_sent_bytes.rate
         - k8s.istio_tcp_received_bytes.rate
@@ -339,11 +342,21 @@ processors:
     metrics:
       - k8s.istio_request_bytes.rate
       - k8s.istio_response_bytes.rate
-      - k8s.istio_request_duration_milliseconds.rate
+      - k8s.istio_request_duration_milliseconds_sum_temp
+      - k8s.istio_request_duration_milliseconds_count_temp
       - k8s.istio_requests.rate
       - k8s.istio_tcp_sent_bytes.rate
       - k8s.istio_tcp_received_bytes.rate
 
+  experimental_metricsgeneration/istio-metrics:
+    rules:
+      - name: k8s.istio_request_duration_milliseconds.rate
+        type: calculate
+        metric1: k8s.istio_request_duration_milliseconds_sum_temp
+        metric2: k8s.istio_request_duration_milliseconds_count_temp
+        operation: divide
+
+
 connectors:
 {{- if and .Values.otel.metrics.enabled (or (not .Values.aws_fargate.enabled) .Values.otel.metrics.autodiscovery.prometheusEndpoints.enabled) }}
   forward/metric-exporter:
@@ -634,11 +647,15 @@ service:
         - metricstransform/istio-metrics
         - cumulativetodelta/istio-metrics
         - deltatorate/istio-metrics
+        - experimental_metricsgeneration/istio-metrics
 {{- if .Values.otel.metrics.autodiscovery.prometheusEndpoints.customTransformations.counterToRate }}
         - cumulativetodelta/discovery
         - deltatorate/discovery
 {{- end }}
         - groupbyattrs/common-all
+{{- if .Values.otel.metrics.filter }}
+        - filter/metrics
+{{- end }}
         - resource/all
       receivers:
         - receiver_creator/discovery

diff --git a/deploy/helm/tests/__snapshot__/logs-fargate-config-map_test.yaml.snap b/deploy/helm/tests/__snapshot__/logs-fargate-config-map_test.yaml.snap
@@ -24,7 +24,7 @@ Fargate logging ConfigMap spec should include additional filters when they are c
           Match *
           Add sw.k8s.cluster.uid <CLUSTER_UID>
           Add sw.k8s.log.type container
-          Add sw.k8s.agent.manifest.version "3.2.0-alpha.16"
+          Add sw.k8s.agent.manifest.version "3.2.0-alpha.17"
     flb_log_cw: "false"
     output.conf: |
       [OUTPUT]
@@ -64,7 +64,7 @@ Fargate logging ConfigMap spec should match snapshot when Fargate logging is ena
           Match *
           Add sw.k8s.cluster.uid <CLUSTER_UID>
           Add sw.k8s.log.type container
-          Add sw.k8s.agent.manifest.version "3.2.0-alpha.16"
+          Add sw.k8s.agent.manifest.version "3.2.0-alpha.17"
     flb_log_cw: "false"
     output.conf: |
       [OUTPUT]

diff --git a/deploy/helm/tests/__snapshot__/node-collector-config-map-windows_test.yaml.snap b/deploy/helm/tests/__snapshot__/node-collector-config-map-windows_test.yaml.snap
@@ -104,7 +104,8 @@ Node collector config for windows nodes should match snapshot when using default
             metrics:
             - k8s.istio_request_bytes.rate
             - k8s.istio_response_bytes.rate
-            - k8s.istio_request_duration_milliseconds.rate
+            - k8s.istio_request_duration_milliseconds_sum_temp
+            - k8s.istio_request_duration_milliseconds_count_temp
             - k8s.istio_requests.rate
             - k8s.istio_tcp_sent_bytes.rate
             - k8s.istio_tcp_received_bytes.rate
@@ -146,10 +147,18 @@ Node collector config for windows nodes should match snapshot when using default
           metrics:
           - k8s.istio_request_bytes.rate
           - k8s.istio_response_bytes.rate
-          - k8s.istio_request_duration_milliseconds.rate
+          - k8s.istio_request_duration_milliseconds_sum_temp
+          - k8s.istio_request_duration_milliseconds_count_temp
           - k8s.istio_requests.rate
           - k8s.istio_tcp_sent_bytes.rate
           - k8s.istio_tcp_received_bytes.rate
+        experimental_metricsgeneration/istio-metrics:
+          rules:
+          - metric1: k8s.istio_request_duration_milliseconds_sum_temp
+            metric2: k8s.istio_request_duration_milliseconds_count_temp
+            name: k8s.istio_request_duration_milliseconds.rate
+            operation: divide
+            type: calculate
         filter/histograms:
           metrics:
             metric:
@@ -889,12 +898,16 @@ Node collector config for windows nodes should match snapshot when using default
             statements:
             - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name
               == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+            - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
             - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
             - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-            - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
             - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
             - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
             - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+            - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name ==
+              "k8s.istio_request_duration_milliseconds_sum"
+            - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name
+              == "k8s.istio_request_duration_milliseconds_count"
         transform/syslogify:
           error_mode: ignore
           log_statements:
@@ -1111,7 +1124,9 @@ Node collector config for windows nodes should match snapshot when using default
             - metricstransform/istio-metrics
             - cumulativetodelta/istio-metrics
             - deltatorate/istio-metrics
+            - experimental_metricsgeneration/istio-metrics
             - groupbyattrs/common-all
+            - filter/metrics
             - resource/all
             receivers:
             - receiver_creator/discovery

diff --git a/deploy/helm/tests/__snapshot__/node-collector-config-map_test.yaml.snap b/deploy/helm/tests/__snapshot__/node-collector-config-map_test.yaml.snap
@@ -104,7 +104,8 @@ Node collector config should match snapshot when autodiscovery is disabled:
             metrics:
             - k8s.istio_request_bytes.rate
             - k8s.istio_response_bytes.rate
-            - k8s.istio_request_duration_milliseconds.rate
+            - k8s.istio_request_duration_milliseconds_sum_temp
+            - k8s.istio_request_duration_milliseconds_count_temp
             - k8s.istio_requests.rate
             - k8s.istio_tcp_sent_bytes.rate
             - k8s.istio_tcp_received_bytes.rate
@@ -146,10 +147,18 @@ Node collector config should match snapshot when autodiscovery is disabled:
           metrics:
           - k8s.istio_request_bytes.rate
           - k8s.istio_response_bytes.rate
-          - k8s.istio_request_duration_milliseconds.rate
+          - k8s.istio_request_duration_milliseconds_sum_temp
+          - k8s.istio_request_duration_milliseconds_count_temp
           - k8s.istio_requests.rate
           - k8s.istio_tcp_sent_bytes.rate
           - k8s.istio_tcp_received_bytes.rate
+        experimental_metricsgeneration/istio-metrics:
+          rules:
+          - metric1: k8s.istio_request_duration_milliseconds_sum_temp
+            metric2: k8s.istio_request_duration_milliseconds_count_temp
+            name: k8s.istio_request_duration_milliseconds.rate
+            operation: divide
+            type: calculate
         filter/histograms:
           metrics:
             metric:
@@ -909,12 +918,16 @@ Node collector config should match snapshot when autodiscovery is disabled:
             statements:
             - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name
               == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+            - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
             - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
             - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-            - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
             - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
             - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
             - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+            - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name ==
+              "k8s.istio_request_duration_milliseconds_sum"
+            - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name
+              == "k8s.istio_request_duration_milliseconds_count"
         transform/syslogify:
           error_mode: ignore
           log_statements:
@@ -1250,7 +1263,8 @@ Node collector config should match snapshot when fargate is enabled:
             metrics:
             - k8s.istio_request_bytes.rate
             - k8s.istio_response_bytes.rate
-            - k8s.istio_request_duration_milliseconds.rate
+            - k8s.istio_request_duration_milliseconds_sum_temp
+            - k8s.istio_request_duration_milliseconds_count_temp
             - k8s.istio_requests.rate
             - k8s.istio_tcp_sent_bytes.rate
             - k8s.istio_tcp_received_bytes.rate
@@ -1292,10 +1306,18 @@ Node collector config should match snapshot when fargate is enabled:
           metrics:
           - k8s.istio_request_bytes.rate
           - k8s.istio_response_bytes.rate
-          - k8s.istio_request_duration_milliseconds.rate
+          - k8s.istio_request_duration_milliseconds_sum_temp
+          - k8s.istio_request_duration_milliseconds_count_temp
           - k8s.istio_requests.rate
           - k8s.istio_tcp_sent_bytes.rate
           - k8s.istio_tcp_received_bytes.rate
+        experimental_metricsgeneration/istio-metrics:
+          rules:
+          - metric1: k8s.istio_request_duration_milliseconds_sum_temp
+            metric2: k8s.istio_request_duration_milliseconds_count_temp
+            name: k8s.istio_request_duration_milliseconds.rate
+            operation: divide
+            type: calculate
         filter/histograms:
           metrics:
             metric:
@@ -2055,12 +2077,16 @@ Node collector config should match snapshot when fargate is enabled:
             statements:
             - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name
               == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+            - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
             - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
             - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-            - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
             - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
             - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
             - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+            - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name ==
+              "k8s.istio_request_duration_milliseconds_sum"
+            - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name
+              == "k8s.istio_request_duration_milliseconds_count"
         transform/syslogify:
           error_mode: ignore
           log_statements:
@@ -2254,7 +2280,9 @@ Node collector config should match snapshot when fargate is enabled:
             - metricstransform/istio-metrics
             - cumulativetodelta/istio-metrics
             - deltatorate/istio-metrics
+            - experimental_metricsgeneration/istio-metrics
             - groupbyattrs/common-all
+            - filter/metrics
             - resource/all
             receivers:
             - receiver_creator/discovery
@@ -2363,7 +2391,8 @@ Node collector config should match snapshot when fargate is enabled and autodisc
             metrics:
             - k8s.istio_request_bytes.rate
             - k8s.istio_response_bytes.rate
-            - k8s.istio_request_duration_milliseconds.rate
+            - k8s.istio_request_duration_milliseconds_sum_temp
+            - k8s.istio_request_duration_milliseconds_count_temp
             - k8s.istio_requests.rate
             - k8s.istio_tcp_sent_bytes.rate
             - k8s.istio_tcp_received_bytes.rate
@@ -2405,10 +2434,18 @@ Node collector config should match snapshot when fargate is enabled and autodisc
           metrics:
           - k8s.istio_request_bytes.rate
           - k8s.istio_response_bytes.rate
-          - k8s.istio_request_duration_milliseconds.rate
+          - k8s.istio_request_duration_milliseconds_sum_temp
+          - k8s.istio_request_duration_milliseconds_count_temp
           - k8s.istio_requests.rate
           - k8s.istio_tcp_sent_bytes.rate
           - k8s.istio_tcp_received_bytes.rate
+        experimental_metricsgeneration/istio-metrics:
+          rules:
+          - metric1: k8s.istio_request_duration_milliseconds_sum_temp
+            metric2: k8s.istio_request_duration_milliseconds_count_temp
+            name: k8s.istio_request_duration_milliseconds.rate
+            operation: divide
+            type: calculate
         filter/histograms:
           metrics:
             metric:
@@ -3168,12 +3205,16 @@ Node collector config should match snapshot when fargate is enabled and autodisc
             statements:
             - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name
               == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+            - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
             - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
             - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-            - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
             - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
             - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
             - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+            - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name ==
+              "k8s.istio_request_duration_milliseconds_sum"
+            - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name
+              == "k8s.istio_request_duration_milliseconds_count"
         transform/syslogify:
           error_mode: ignore
           log_statements:
@@ -3440,7 +3481,8 @@ Node collector config should match snapshot when using default values:
             metrics:
             - k8s.istio_request_bytes.rate
             - k8s.istio_response_bytes.rate
-            - k8s.istio_request_duration_milliseconds.rate
+            - k8s.istio_request_duration_milliseconds_sum_temp
+            - k8s.istio_request_duration_milliseconds_count_temp
             - k8s.istio_requests.rate
             - k8s.istio_tcp_sent_bytes.rate
             - k8s.istio_tcp_received_bytes.rate
@@ -3482,10 +3524,18 @@ Node collector config should match snapshot when using default values:
           metrics:
           - k8s.istio_request_bytes.rate
           - k8s.istio_response_bytes.rate
-          - k8s.istio_request_duration_milliseconds.rate
+          - k8s.istio_request_duration_milliseconds_sum_temp
+          - k8s.istio_request_duration_milliseconds_count_temp
           - k8s.istio_requests.rate
           - k8s.istio_tcp_sent_bytes.rate
           - k8s.istio_tcp_received_bytes.rate
+        experimental_metricsgeneration/istio-metrics:
+          rules:
+          - metric1: k8s.istio_request_duration_milliseconds_sum_temp
+            metric2: k8s.istio_request_duration_milliseconds_count_temp
+            name: k8s.istio_request_duration_milliseconds.rate
+            operation: divide
+            type: calculate
         filter/histograms:
           metrics:
             metric:
@@ -4245,12 +4295,16 @@ Node collector config should match snapshot when using default values:
             statements:
             - extract_sum_metric(true) where (name == "k8s.istio_request_bytes" or name
               == "k8s.istio_response_bytes" or name == "k8s.istio_request_duration_milliseconds")
+            - extract_count_metric(true) where (name == "k8s.istio_request_duration_milliseconds")
             - set(name, "k8s.istio_request_bytes.rate") where name == "k8s.istio_request_bytes_sum"
             - set(name, "k8s.istio_response_bytes.rate") where name == "k8s.istio_response_bytes_sum"
-            - set(name, "k8s.istio_request_duration_milliseconds.rate") where name == "k8s.istio_request_duration_milliseconds_sum"
             - set(name, "k8s.istio_requests.rate") where name == "k8s.istio_requests_total"
             - set(name, "k8s.istio_tcp_sent_bytes.rate") where name == "k8s.istio_tcp_sent_bytes_total"
             - set(name, "k8s.istio_tcp_received_bytes.rate") where name == "k8s.istio_tcp_received_bytes_total"
+            - set(name, "k8s.istio_request_duration_milliseconds_sum_temp") where name ==
+              "k8s.istio_request_duration_milliseconds_sum"
+            - set(name, "k8s.istio_request_duration_milliseconds_count_temp") where name
+              == "k8s.istio_request_duration_milliseconds_count"
         transform/syslogify:
           error_mode: ignore
           log_statements:
@@ -4481,7 +4535,9 @@ Node collector config should match snapshot when using default values:
             - metricstransform/istio-metrics
             - cumulativetodelta/istio-metrics
             - deltatorate/istio-metrics
+            - experimental_metricsgeneration/istio-metrics
             - groupbyattrs/common-all
+            - filter/metrics
             - resource/all
             receivers:
             - receiver_creator/discovery