From 13f1d50f2ed6168b1166213cbcebacba9eb66093 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Mon, 12 Aug 2024 15:06:08 -0600 Subject: [PATCH] Added initial metrics dashboard and dev containers to the dev docker-compose stack, so it includes a preconfigured Grafana + Prometheus services, with provisioned dashboard on source. Added documentation for metrics as well. Signed-off-by: Alfredo Gutierrez --- server/docker/docker-compose.yml | 28 +- .../metrics/dashboards/block-node-server.json | 555 ++++++++++++++++++ .../metrics/dashboards/provisioner.yaml | 11 + .../metrics/datasources/datasources.yml | 12 + server/docker/metrics/prometheus.yml | 7 + server/docs/metrics.md | 23 +- .../java/com/hedera/block/server/Server.java | 7 +- .../config/BlockNodeContextFactory.java | 33 +- .../block/server/metrics/MetricsService.java | 15 - 9 files changed, 659 insertions(+), 32 deletions(-) create mode 100644 server/docker/metrics/dashboards/block-node-server.json create mode 100644 server/docker/metrics/dashboards/provisioner.yaml create mode 100644 server/docker/metrics/datasources/datasources.yml create mode 100644 server/docker/metrics/prometheus.yml diff --git a/server/docker/docker-compose.yml b/server/docker/docker-compose.yml index 57740349..72d5db2f 100644 --- a/server/docker/docker-compose.yml +++ b/server/docker/docker-compose.yml @@ -6,6 +6,32 @@ services: env_file: - .env ports: - - "9999:9999" - "8080:8080" - "5005:5005" + - "9999:9999" + + prometheus: + image: prom/prometheus + container_name: prometheus + volumes: + - ./metrics/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - "9090:9090" + + grafana: + image: grafana/grafana + container_name: grafana + depends_on: + - prometheus + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + volumes: + - ./metrics/dashboards:/etc/grafana/provisioning/dashboards + - ./metrics/datasources/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + ports: + - "3000:3000" + diff --git a/server/docker/metrics/dashboards/block-node-server.json b/server/docker/metrics/dashboards/block-node-server.json new file mode 100644 index 00000000..f32013ac --- /dev/null +++ b/server/docker/metrics/dashboards/block-node-server.json @@ -0,0 +1,555 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Live Stream", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_live_block_items_total", + "instant": false, + "legendFormat": "BlockItems", + "range": true, + "refId": "A" + } + ], + "title": "Live Block Item Counter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "#EAB839", + "value": 5 + }, + { + "color": "green", + "value": 30 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate( hedera_block_node_live_block_items_total [$__rate_interval])", + "instant": false, + "legendFormat": "BlockItems", + "range": true, + "refId": "A" + } + ], + "title": "Live Block Item Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_blocks_persisted_total", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Block Persistence Counter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 8 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_blocks_persisted_total[$__rate_interval])", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Block Persistence Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Subscribers", + "mappings": [], + "max": 20, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 15 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 0, + "y": 15 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_subscribers", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Subscribers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "instant": false, + "legendFormat": "RPS of Single Blocks Retrieval", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Single Blocks Retrieval", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Block-Node Server Dashboard", + "uid": "edu86nutnxts0c", + "version": 1, + "weekStart": "" +} diff --git a/server/docker/metrics/dashboards/provisioner.yaml b/server/docker/metrics/dashboards/provisioner.yaml new file mode 100644 index 00000000..006b093e --- /dev/null +++ b/server/docker/metrics/dashboards/provisioner.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/server/docker/metrics/datasources/datasources.yml b/server/docker/metrics/datasources/datasources.yml new file mode 100644 index 00000000..b2a9b892 --- /dev/null +++ b/server/docker/metrics/datasources/datasources.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: PBFA97CFB590B2093 + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + timeInterval: 1s diff --git a/server/docker/metrics/prometheus.yml b/server/docker/metrics/prometheus.yml new file mode 100644 index 00000000..2a2d1cc7 --- /dev/null +++ b/server/docker/metrics/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 1s + +scrape_configs: + - job_name: 'local' + static_configs: + - targets: ['block-node-server:9999'] diff --git a/server/docs/metrics.md b/server/docs/metrics.md index 46cd3ee5..50644807 100644 --- a/server/docs/metrics.md +++ b/server/docs/metrics.md @@ -29,9 +29,24 @@ MetricsService centralizes the creation of metrics and provides a way to access To check the metrics you can access the Prometheus endpoint at `http://localhost:9999/metrics`. +### Local Development +For developers, when using the gradle task `startDockerContainer` it will automatically start a prometheus and grafana services preconfigured locally with credentials: username `admin` and paassword `admin` and the dashboard already provisioned with the current metrics and widgets. + +Dashboard is called `Block-Node Server Dashboard` and its source is kept on folder: `server/docker/metrics/dashboards` as: `block-node-server.json`. + +When doing changes to the dashboard on grafana is important to copy the json to clipboard and commit the changes on the above file, so dashboard is updated with the latest widgets for the new metrics added. + +If needed to create another dashboard is possible to include it by adding it to the same folder. + + ## Existing Metrics -| Metric Name | Description | Type | -|----------------|---------------------------|---------| -| exampleGauge | An example gauge metric | Gauge | -| exampleCounter | An example counter metric | Counter | +All metrics have `hedera_block_node` prefix. + +| Metric Name | Description | Type | +|---------------------------------|---------------------------------------|-----------| +| live_block_items | The amount of block items received | Counter | +| blocks_persisted | the amount of blocks persisted | Counter | +| subscribers | The amount of subscribers | Gauge | +| single_blocks_retrieved | the amount of single blocks requested | Counter | + diff --git a/server/src/main/java/com/hedera/block/server/Server.java b/server/src/main/java/com/hedera/block/server/Server.java index 0c392c59..980c8efd 100644 --- a/server/src/main/java/com/hedera/block/server/Server.java +++ b/server/src/main/java/com/hedera/block/server/Server.java @@ -25,7 +25,6 @@ import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; import com.hedera.block.server.mediator.StreamMediator; -import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; @@ -54,13 +53,9 @@ public static void main(final String[] args) { LOGGER.log(System.Logger.Level.INFO, "Starting BlockNode Server"); try { - // init metrics + // init context, metrics, and configuration. @NonNull final BlockNodeContext blockNodeContext = BlockNodeContextFactory.create(); - // increase by 1 just for the sake of an example - @NonNull final MetricsService metricsService = blockNodeContext.metricsService(); - metricsService.exampleCounter.increment(); - // Set the global configuration @NonNull final Config config = Config.create(); Config.global(config); diff --git a/server/src/main/java/com/hedera/block/server/config/BlockNodeContextFactory.java b/server/src/main/java/com/hedera/block/server/config/BlockNodeContextFactory.java index b47098be..9081b770 100644 --- a/server/src/main/java/com/hedera/block/server/config/BlockNodeContextFactory.java +++ b/server/src/main/java/com/hedera/block/server/config/BlockNodeContextFactory.java @@ -28,21 +28,33 @@ import java.io.IOException; import java.nio.file.Path; -/** Static factory that creates {@link BlockNodeContext} */ +/** + * Use the static method create() to obtain a new {@link BlockNodeContext} when initializing the + * application. + * + *

When a context is created all enabled sources of configuration information will be read and + * applicable values made available through the context created.
+ * The application should use the context to obtain configuration and metrics support. + */ public class BlockNodeContextFactory { - private static final String APPLICATION_PROPERTIES_1 = "app.properties"; + /** + * A resource file name from which application configuration is read. The properties in this + * file are available as configuration from the {@link BlockNodeContext}. + */ + private static final String APPLICATION_PROPERTIES = "app.properties"; private BlockNodeContextFactory() {} /** - * Use the create method to build a singleton block node context to manage system-wide metrics. + * Create a new application context for use in the system. The context is needed at + * initialization. * - * @return an instance of {@link BlockNodeContext} which holds {@link Configuration}, {@link - * Metrics} and {@link MetricsService} for the rest of the application to use. + * @return a context with configuration and metrics. * @throws IOException when the java libraries fail to read information from a configuration * source. */ + @NonNull public static BlockNodeContext create() throws IOException { final Configuration configuration = getConfiguration(); final Metrics metrics = getMetrics(configuration); @@ -50,12 +62,21 @@ public static BlockNodeContext create() throws IOException { return new BlockNodeContext(metrics, metricsService, configuration); } + /** + * Read configuration for this system. The configuration sources will include environment + * variables, system properties, and the properties file named {@value APPLICATION_PROPERTIES}. + * + * @return the configuration as read from environment, properties, and files. + * @throws IOException when the java libraries fail to read information from a configuration + * source. + */ + @NonNull private static Configuration getConfiguration() throws IOException { return ConfigurationBuilder.create() .withSource(SystemEnvironmentConfigSource.getInstance()) .withSource(SystemPropertiesConfigSource.getInstance()) - .withSource(new ClasspathFileConfigSource(Path.of(APPLICATION_PROPERTIES_1))) + .withSource(new ClasspathFileConfigSource(Path.of(APPLICATION_PROPERTIES))) .autoDiscoverExtensions() .build(); } diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java index 4dd4a59a..a383d8a0 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java @@ -31,12 +31,6 @@ public class MetricsService { private static final String CATEGORY = "hedera_block_node"; - private static final LongGauge.Config EXAMPLE_GAUGE = - new LongGauge.Config(CATEGORY, "exampleGauge").withDescription("An example gauge"); - - private static final Counter.Config EXAMPLE_COUNTER = - new Counter.Config(CATEGORY, "exampleCounter").withDescription("An example counter"); - // Live BlockItem Counter private static final Counter.Config LIVE_BLOCK_ITEM_COUNTER = new Counter.Config(CATEGORY, "live_block_items").withDescription("Live BlockItems"); @@ -54,12 +48,6 @@ public class MetricsService { new Counter.Config(CATEGORY, "single_blocks_retrieved") .withDescription("Single Blocks Retrieved"); - /** An example gauge. */ - public final LongGauge exampleGauge; - - /** An example counter. */ - public final Counter exampleCounter; - /** Update the counter of live block items transiting via the live stream. */ public final Counter liveBlockItems; @@ -78,9 +66,6 @@ public class MetricsService { * @param metrics the metrics instance */ public MetricsService(@NonNull final Metrics metrics) { - this.exampleGauge = metrics.getOrCreate(EXAMPLE_GAUGE); - this.exampleCounter = metrics.getOrCreate(EXAMPLE_COUNTER); - this.liveBlockItems = metrics.getOrCreate(LIVE_BLOCK_ITEM_COUNTER); this.blocksPersisted = metrics.getOrCreate(BLOCK_PERSISTENCE_COUNTER); this.singleBlocksRetrieved = metrics.getOrCreate(SINGLE_BLOCK_RETRIEVED_COUNTER);