diff --git a/Cargo.lock b/Cargo.lock index 5d408c1..879db4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,6 +917,7 @@ dependencies = [ "canister-utils", "dashmap", "ic-agent", + "metrics", "tokio", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 77ea7c4..1cc0b7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ serde = "1.0.176" reqwest = "0.11.7" tokio = { version = "1.29.1", features = ["full"] } tracing = "0.1.40" +metrics = "0.22.1" canister-utils = { path = "src/canister-utils" } ic-identity = { path = "src/ic-identity" } diff --git a/src/gateway-state/Cargo.toml b/src/gateway-state/Cargo.toml index 1220a21..ecf3f94 100644 --- a/src/gateway-state/Cargo.toml +++ b/src/gateway-state/Cargo.toml @@ -13,4 +13,5 @@ dashmap = "5.5.3" tokio = { workspace = true } tracing = { workspace = true } +metrics = { workspace = true } canister-utils = { workspace = true } diff --git a/src/gateway-state/src/lib.rs b/src/gateway-state/src/lib.rs index 4d25d1b..ddd2bce 100644 --- a/src/gateway-state/src/lib.rs +++ b/src/gateway-state/src/lib.rs @@ -1,9 +1,12 @@ -use canister_utils::{ClientKey, IcWsCanisterMessage}; +use std::sync::Arc; + use dashmap::{mapref::entry::Entry, DashMap}; use ic_agent::export::Principal; -use std::sync::Arc; +use metrics::gauge; use tokio::sync::mpsc::Sender; -use tracing::Span; +use tracing::{debug, Span}; + +use canister_utils::{ClientKey, IcWsCanisterMessage}; /// State of the WS Gateway that can be shared between threads #[derive(Clone)] @@ -59,6 +62,12 @@ impl GatewayState { span: client_session_span, }, ); + + // Increment the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // the poller shall not be started again None }, @@ -74,6 +83,12 @@ impl GatewayState { }, ); entry.insert(Arc::clone(&poller_state)); + + // Increment the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // the poller shall be started Some(poller_state) }, @@ -104,6 +119,12 @@ impl GatewayState { // if this is encountered it might indicate a race condition unreachable!("Client key not found in poller state"); } + + // Decrement the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); // even if this is the last client session for the canister, do not remove the canister from the gateway state // this will be done by the poller task } @@ -115,14 +136,6 @@ impl GatewayState { // therefore there is no need to do anything else here } - pub fn get_clients_count(&self, canister_id: CanisterPrincipal) -> usize { - if let Some(poller_state) = self.inner.data.get(&canister_id) { - poller_state.len() - } else { - 0 - } - } - pub fn get_active_pollers_count(&self) -> usize { self.inner.data.len() } @@ -155,7 +168,15 @@ impl GatewayState { // returns 'ClientRemovalResult::Removed' if the client was removed, 'ClientRemovalResult::Vacant' if there was no such client return { match poller_state.remove(&client_key) { - Some(_) => ClientRemovalResult::Removed(client_key), + Some(_) => { + // Decrement the number of clients connected to the canister + let clients_connected = poller_state.len(); + debug!("Clients connected: {}", clients_connected); + gauge!("clients_connected", "canister_id" => canister_id.to_string()) + .set(clients_connected as f64); + + ClientRemovalResult::Removed(client_key) + }, None => ClientRemovalResult::Vacant, } }; @@ -262,14 +283,15 @@ pub type CanisterPrincipal = Principal; #[cfg(test)] mod tests { - use tokio::sync::mpsc::{self, Receiver}; - - use super::*; use std::{ thread, time::{Duration, Instant}, }; + use tokio::sync::mpsc::{self, Receiver}; + + use super::*; + #[tokio::test] async fn should_insert_new_client_channels_and_get_new_poller_state_once() { let clients_count = 1000; diff --git a/src/ic-websocket-gateway/Cargo.toml b/src/ic-websocket-gateway/Cargo.toml index 534b232..91a7094 100644 --- a/src/ic-websocket-gateway/Cargo.toml +++ b/src/ic-websocket-gateway/Cargo.toml @@ -36,7 +36,7 @@ opentelemetry = { version = "0.21" } opentelemetry-otlp = { version = "0.14.0" } opentelemetry_sdk = { version = "0.21", features = ["rt-tokio"] } rand = "0.8" -metrics = "0.22.1" +metrics = { workspace = true } metrics-exporter-prometheus = "0.13.1" metrics-util = "0.16.2" diff --git a/src/ic-websocket-gateway/src/canister_poller.rs b/src/ic-websocket-gateway/src/canister_poller.rs index 4fd2f01..8bc1f3a 100644 --- a/src/ic-websocket-gateway/src/canister_poller.rs +++ b/src/ic-websocket-gateway/src/canister_poller.rs @@ -7,6 +7,7 @@ use gateway_state::{ CanisterPrincipal, CanisterRemovalResult, ClientSender, GatewayState, PollerState, }; use ic_agent::{agent::RejectCode, Agent, AgentError}; +use metrics::{gauge, histogram}; use std::{sync::Arc, time::Duration}; use tokio::{sync::mpsc::Sender, time::timeout}; use tracing::{error, span, trace, warn, Instrument, Level, Span}; @@ -77,6 +78,12 @@ impl CanisterPoller { // this enables to crawl polling iterations in reverse chronological order polling_iteration_span.follows_from(previous_polling_iteration_span.id()); } + + // register the number of active clients + let clients_connected = self.poller_state.len(); + gauge!("clients_connected", "canister_id" => self.canister_id.to_string()) + .set(clients_connected as f64); + if let Err(e) = self .poll_and_relay() .instrument(polling_iteration_span.clone()) @@ -140,7 +147,10 @@ impl CanisterPoller { PollingStatus::NoMessagesPolled => (), } - // compute the amout of time to sleep for before polling again + // record the time it took to poll the canister + let delta = start_polling_instant.elapsed(); + histogram!("poller_duration", "canister_id" => self.canister_id.to_string()).record(delta); + let effective_polling_interval = self.compute_effective_polling_interval(start_polling_instant); // if no messages are returned or if the queue is fully drained, sleep for 'effective_polling_interval' before polling again diff --git a/src/ic-websocket-gateway/src/client_session_handler.rs b/src/ic-websocket-gateway/src/client_session_handler.rs index 7a35d63..5b99e41 100644 --- a/src/ic-websocket-gateway/src/client_session_handler.rs +++ b/src/ic-websocket-gateway/src/client_session_handler.rs @@ -7,7 +7,7 @@ use canister_utils::{ws_close, CanisterWsCloseArguments, ClientKey, IcWsCanister use futures_util::StreamExt; use gateway_state::{CanisterPrincipal, ClientRemovalResult, GatewayState, PollerState}; use ic_agent::Agent; -use metrics::{gauge, histogram}; +use metrics::{counter, gauge, histogram}; use std::sync::Arc; use std::time::Instant; use tokio::{ @@ -189,14 +189,9 @@ impl ClientSessionHandler { client_session_span.in_scope(|| { debug!("Client session opened"); - let canister_id = self.get_canister_id(&client_session); let client_key = self.get_client_key(&client_session); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()).set(clients_connected as f64); - + counter!("client_connected_count", "client_key" => client_key.to_string()).absolute(1); // Calculate the time it took to open the connection and record it using the timer started in ws_listener.rs let delta = self.start_connection_time.elapsed(); histogram!("connection_opening_time", "client_key" => client_key.to_string()).record(delta); @@ -215,12 +210,6 @@ impl ClientSessionHandler { .remove_client(canister_id, client_key.clone()); debug!("Client removed from gateway state"); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()) - .set(clients_connected as f64); - let delta = client_start_session_time.elapsed(); histogram!("connection_duration", "client_key" => client_key.to_string()) .record(delta); @@ -256,12 +245,6 @@ impl ClientSessionHandler { { debug!("Client removed from gateway state"); - // Clients connection metrics - let clients_connected = self.gateway_state.get_clients_count(canister_id); - debug!("Clients connected: {}", clients_connected.to_string()); - gauge!("clients_connected", "canister_id" => canister_id.to_string()) - .set(clients_connected as f64); - let delta = client_start_session_time.elapsed(); histogram!("connection_duration", "client_key" => client_key.to_string()) .record(delta); diff --git a/src/ic-websocket-gateway/src/gateway_metrics.rs b/src/ic-websocket-gateway/src/gateway_metrics.rs index f2a08b3..aad3ea5 100644 --- a/src/ic-websocket-gateway/src/gateway_metrics.rs +++ b/src/ic-websocket-gateway/src/gateway_metrics.rs @@ -1,4 +1,4 @@ -use metrics::{describe_gauge, describe_histogram, gauge}; +use metrics::{describe_counter, describe_gauge, describe_histogram, gauge}; use metrics_exporter_prometheus::PrometheusBuilder; use metrics_util::MetricKindMask; use std::error::Error; @@ -9,12 +9,16 @@ use std::time::Duration; pub fn init_metrics(address: &str) -> Result<(), Box> { let builder = PrometheusBuilder::new().with_http_listener(SocketAddr::from_str(address)?); - // Set the idle timeout for counters and histograms to 30 seconds then the metrics are removed from the registry + // Set the idle timeout for counters and histograms to 10 seconds then the metrics are removed from the registry builder - .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(30))) + .idle_timeout(MetricKindMask::ALL, Some(Duration::from_secs(10))) .install() .expect("failed to install Prometheus recorder"); + describe_counter!( + "client_connected_count", + "Each time that a client connects it emits a point" + ); describe_gauge!( "clients_connected", "The number of clients currently connected" @@ -32,6 +36,7 @@ pub fn init_metrics(address: &str) -> Result<(), Box> { "connection_opening_time", "The time it takes to open a connection" ); + describe_histogram!("poller_duration", "The time it takes to poll the canister"); gauge!("active_pollers").set(0.0); diff --git a/telemetry/dashboards/ic_metrics.json b/telemetry/dashboards/ic_metrics.json index 2eafcd6..6893b2e 100644 --- a/telemetry/dashboards/ic_metrics.json +++ b/telemetry/dashboards/ic_metrics.json @@ -111,13 +111,221 @@ "title": "Number of active pollers", "type": "gauge" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 14, + "interval": "10s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "poller_duration{quantile=\"1\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{canister_id}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Polling interval by canister id", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average is calculated every 1 minute and aggregated by instance", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 15, + "interval": "10s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg by(instance) (avg_over_time(poller_duration{quantile=\"1\"}[$__interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average polling interval", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 17 }, "id": 9, "panels": [], @@ -193,7 +401,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 18 }, "id": 3, "interval": "10s", @@ -298,7 +506,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 18 }, "id": 4, "interval": "10s", @@ -352,7 +560,7 @@ "axisColorMode": "text", "axisLabel": "Number", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", @@ -402,10 +610,10 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 26 }, "id": 6, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -427,7 +635,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(instance) (max_over_time(clients_connected[30s]))", + "expr": "sum by(instance) (max_over_time(client_connected_count[$__interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -458,7 +666,7 @@ "axisColorMode": "text", "axisLabel": "Number", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", @@ -508,10 +716,10 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 26 }, "id": 7, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -533,7 +741,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(instance) (max_over_time(clients_connected[30s]))", + "expr": "sum by(instance) (max_over_time(client_connected_count[$__interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -613,10 +821,10 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "id": 2, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -717,10 +925,10 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 34 }, "id": 8, - "interval": "30s", + "interval": "10s", "options": { "legend": { "calcs": [], @@ -761,7 +969,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 42 }, "id": 11, "panels": [], @@ -776,7 +984,39 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "max": 1, @@ -801,23 +1041,21 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 12, + "interval": "10s", "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "10.4.0", "targets": [ @@ -839,7 +1077,7 @@ } ], "title": "Gateway status", - "type": "gauge" + "type": "timeseries" }, { "datasource": { @@ -906,9 +1144,10 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 43 }, "id": 13, + "interval": "10s", "options": { "legend": { "calcs": [], @@ -957,6 +1196,6 @@ "timezone": "browser", "title": "IC Metrics", "uid": "cdfh3d16g8dtsf", - "version": 9, + "version": 17, "weekStart": "" } diff --git a/telemetry/dashboards/duration_by_thread_id.json b/telemetry/dashboards/ic_ws_gw.json similarity index 50% rename from telemetry/dashboards/duration_by_thread_id.json rename to telemetry/dashboards/ic_ws_gw.json index 460c7c6..c09c314 100644 --- a/telemetry/dashboards/duration_by_thread_id.json +++ b/telemetry/dashboards/ic_ws_gw.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 12, "links": [], "liveNow": false, "panels": [ @@ -30,11 +30,766 @@ "x": 0, "y": 0 }, - "id": 2, + "id": 14, + "panels": [], + "title": "Proxy Canister", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration of HTTP Canister Proxy", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 16, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "8bcdaa62", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "HTTP Requests", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 15, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "0a5d4dcf", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Client Session" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Past Uptimes of HTTP Canister Proxy", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, "panels": [], - "title": "Duration by Thread ID", + "title": "Polling Iterations by Canister ID", "type": "row" }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "mqci4-raaaa-aaaao-a2fyq-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration for IC WS Demo", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 17, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "92284c4a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "mqci4-raaaa-aaaao-a2fyq-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Messages from IC WS Demo Canister", + "type": "barchart" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0-65283", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Polling Iteration" + ], + "valueType": "string" + }, + { + "id": "0b1aaf5a", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "sswdm-viaaa-aaaap-abugq-cai" + ], + "valueType": "string" + } + ], + "limit": 999, + "queryType": "traceqlSearch", + "refId": "A", + "spss": 100, + "tableType": "traces" + } + ], + "title": "Polling Iteration for sswdm", + "type": "timeseries" + }, { "datasource": { "type": "tempo", @@ -86,10 +841,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 1 + "x": 12, + "y": 26 }, - "id": 1, + "id": 18, "options": { "barRadius": 0, "barWidth": 0.97, @@ -105,10 +860,10 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, - "xField": "Start time", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, @@ -120,7 +875,141 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "003150a8", + "operator": "=", + "scope": "span", + "tag": "canister_id", + "value": [ + "sswdm-viaaa-aaaap-abugq-cai" + ], + "valueType": "string" + }, + { + "id": "service-name", + "operator": "=", + "scope": "resource", + "tag": "service.name", + "value": [ + "ic-ws-gw-3656s" + ], + "valueType": "string" + }, + { + "id": "span-name", + "operator": "=", + "scope": "span", + "tag": "name", + "value": [ + "Canister Message" + ], + "valueType": "string" + } + ], + "limit": 20, + "queryType": "traceqlSearch", + "refId": "A", + "tableType": "traces" + } + ], + "title": "Canister messages from sswdm", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 2, + "panels": [], + "title": "Duration by Thread ID", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.95, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "Start time", + "xTickLabelRotation": 45, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "P214B5B846CF3925F" + }, + "filters": [ + { + "id": "eaaf0a62", "operator": "=", "scope": "span", "tag": "thread.id", @@ -135,7 +1024,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -145,18 +1034,29 @@ "scope": "span", "tag": "name", "value": [ - "Accept Connection" + "Client Session" + ], + "valueType": "string" + }, + { + "id": "88e5980f", + "operator": "!=", + "scope": "span", + "tag": "canister_id", + "value": [ + "iustv-tiaaa-aaaao-a3aga-cai" ], "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], - "title": "Thread ID 2", + "title": "Thread ID 2 (excluding HTTP Proxy)", "type": "barchart" }, { @@ -211,7 +1111,7 @@ "h": 8, "w": 12, "x": 12, - "y": 1 + "y": 35 }, "id": 6, "options": { @@ -229,6 +1129,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -244,7 +1145,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "b8078415", "operator": "=", "scope": "span", "tag": "thread.id", @@ -259,7 +1160,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -274,9 +1175,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -335,7 +1237,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 43 }, "id": 5, "options": { @@ -353,6 +1255,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -368,7 +1271,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "d53c2a88", "operator": "=", "scope": "span", "tag": "thread.id", @@ -383,7 +1286,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -398,9 +1301,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -459,7 +1363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 43 }, "id": 7, "options": { @@ -477,6 +1381,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -492,7 +1397,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "7e6c2a74", "operator": "=", "scope": "span", "tag": "thread.id", @@ -507,7 +1412,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -522,9 +1427,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -583,7 +1489,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 51 }, "id": 3, "options": { @@ -601,6 +1507,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -616,11 +1523,13 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "9d20e8d9", "operator": "=", "scope": "span", "tag": "thread.id", - "value": [], + "value": [ + "6" + ], "valueType": "int" }, { @@ -629,7 +1538,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -644,9 +1553,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -705,7 +1615,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 51 }, "id": 8, "options": { @@ -723,6 +1633,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -738,7 +1649,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "1fef4771", "operator": "=", "scope": "span", "tag": "thread.id", @@ -753,7 +1664,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -768,9 +1679,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -829,7 +1741,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 59 }, "id": 4, "options": { @@ -847,6 +1759,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -862,7 +1775,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "3b6ced2a", "operator": "=", "scope": "span", "tag": "thread.id", @@ -877,7 +1790,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -892,9 +1805,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -953,7 +1867,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 59 }, "id": 9, "options": { @@ -971,6 +1885,7 @@ "showValue": "auto", "stacking": "none", "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -986,7 +1901,7 @@ }, "filters": [ { - "id": "d4ede8da", + "id": "a43b45ff", "operator": "=", "scope": "span", "tag": "thread.id", @@ -1001,7 +1916,7 @@ "scope": "resource", "tag": "service.name", "value": [ - "ic-ws-gw-ipc77" + "ic-ws-gw-3656s" ], "valueType": "string" }, @@ -1016,9 +1931,10 @@ "valueType": "string" } ], - "limit": 20, + "limit": 1000, "queryType": "traceqlSearch", "refId": "A", + "spss": 100, "tableType": "traces" } ], @@ -1026,7 +1942,7 @@ "type": "barchart" } ], - "refresh": false, + "refresh": "", "schemaVersion": 39, "tags": [], "templating": { @@ -1036,12 +1952,11 @@ "from": "now-3h", "to": "now" }, + "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "", - "title": "Client Connections by Thread ID", + "title": "IC WS GW", "uid": "a360ec48-ff96-448b-9711-e3e80d339722", - "version": 3, + "version": 15, "weekStart": "" } - - \ No newline at end of file