diff --git a/README.md b/README.md index da71a04..279a1e5 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ It's essential to implement appropriate security measures when dealing with dyna - [ ] Add device-aware context - [ ] Add UI for debugging users - [x] Fix getRouteFromRequest to also match based on path -- [ ] Add metrics for monitoring +- [x] Add metrics for monitoring - [ ] Better logging See the [open issues](https://github.com/rorylshanks/veriflow/issues) for a full list of proposed features (and known issues). diff --git a/lib/http.js b/lib/http.js index 8143f02..4025d42 100644 --- a/lib/http.js +++ b/lib/http.js @@ -10,7 +10,7 @@ import { getConfig, getRedirectBasepath, getAuthListenPort } from '../util/confi import { pem2jwk } from 'pem-jwk'; import crypto from 'crypto'; import errorpages from '../util/errorpage.js' - +import metrics from '../util/metrics.js' var trusted_ranges = ["loopback"].concat(getConfig().trusted_ranges || []) @@ -85,3 +85,5 @@ app.get(getConfig().jwks_path, (req, res) => { }); app.listen(getAuthListenPort(), 'localhost', () => log.debug("Veriflow HTTP server running")); + +metrics.startMetricsServer() \ No newline at end of file diff --git a/lib/idp.js b/lib/idp.js index 562b8d8..40f408b 100644 --- a/lib/idp.js +++ b/lib/idp.js @@ -3,6 +3,7 @@ import Bossbat from 'bossbat'; import log from '../util/logging.js' import { getConfig } from '../util/config.js' import timestring from 'timestring'; +import metrics from '../util/metrics.js' const idpUpdater = new Bossbat({ connection: redisHelper.getRedisConfig(), @@ -16,13 +17,19 @@ let adapter = importedAdapter.default async function update() { try { + const end = metrics.registry.veriflow_idp_update_duration.startTimer(); var startDate = Date.now() await adapter.runUpdate() var endDate = Date.now() var duration = (endDate - startDate) / 1000 + end() log.info(`Updated users from IDP in ${duration} seconds`) + metrics.registry.veriflow_idp_update_total.inc({ result: "success" }) + metrics.registry.veriflow_idp_last_update_time.setToCurrentTime({ result: "success"}) } catch (error) { log.error({error, details: error.message}) + metrics.registry.veriflow_idp_update_total.inc({ result: "failed" }) + metrics.registry.veriflow_idp_last_update_time.setToCurrentTime({ result: "failed"}) } } diff --git a/package-lock.json b/package-lock.json index fedc9a3..cc563be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "pem-jwk": "^2.0.0", "picomatch": "^4.0.1", "pino": "^8.19.0", + "prom-client": "^15.1.0", "timestring": "^7.0.0" }, "devDependencies": { @@ -77,6 +78,14 @@ } } }, + "node_modules/@opentelemetry/api": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.8.0.tgz", + "integrity": "sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -269,6 +278,11 @@ "node": ">=8" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -1871,6 +1885,18 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, + "node_modules/prom-client": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.0.tgz", + "integrity": "sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw==", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2212,6 +2238,14 @@ "node": ">=4" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/thread-stream": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.3.0.tgz", diff --git a/package.json b/package.json index 2a9d412..b805f97 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "pem-jwk": "^2.0.0", "picomatch": "^4.0.1", "pino": "^8.19.0", + "prom-client": "^15.1.0", "timestring": "^7.0.0" }, "devDependencies": { diff --git a/util/caddyModels.js b/util/caddyModels.js index 40b7b49..5d64e8d 100644 --- a/util/caddyModels.js +++ b/util/caddyModels.js @@ -419,7 +419,7 @@ async function generateCaddyConfig() { "http_port": config.data_listen_port, "https_port": 2443, "servers": { - "srv0": { + "veriflow": { "listen": [ ":" + config.data_listen_port ], @@ -430,7 +430,8 @@ async function generateCaddyConfig() { "trusted_proxies": { "ranges": config.trusted_ranges || [], "source": "static" - } + }, + "metrics": {} } } } diff --git a/util/config.js b/util/config.js index 300965d..2112e88 100644 --- a/util/config.js +++ b/util/config.js @@ -4,6 +4,7 @@ import fsSync from 'fs'; import log from './logging.js'; import reloadCaddy from './caddyModels.js'; import chokidar from 'chokidar'; +import metrics from './metrics.js' let foundPort = false @@ -23,8 +24,12 @@ async function reloadConfig() { currentConfig = tempConfig } reloadCaddy.generateCaddyConfig() + metrics.registry.veriflow_config_reloads_total.inc({result: "success"}) } catch (error) { log.error({ message: "Failed to reload config", context: {error: error.message, stack: error.stack}}) + metrics.registry.veriflow_config_reloads_total.inc({result: "failed"}) + } finally { + metrics.registry.veriflow_config_last_reload_time.setToCurrentTime() } } diff --git a/util/metrics.js b/util/metrics.js new file mode 100644 index 0000000..d8c7d71 --- /dev/null +++ b/util/metrics.js @@ -0,0 +1,75 @@ +import log from './logging.js'; +import * as client from 'prom-client'; +import axios from 'axios'; +import { getConfig } from "./config.js" +import express from 'express'; + +const app = express(); + +const collectDefaultMetrics = client.collectDefaultMetrics; +collectDefaultMetrics(); + +const veriflow_config_reloads_total = new client.Counter({ + name: 'veriflow_config_reloads_total', + help: 'Number of configuration reloads and their results (failed/success)', + labelNames: ['result'] +}); + +const veriflow_config_last_reload_time = new client.Gauge({ + name: 'veriflow_config_last_reload_time', + help: 'Time of last config reload' +}); + +const veriflow_idp_update_duration = new client.Gauge({ + name: 'veriflow_idp_update_duration', + help: 'Duration of the last IdP update' +}); + +const veriflow_idp_update_total = new client.Counter({ + name: 'veriflow_idp_update_total', + help: 'Number of IdP updates and their result (failed/success)', + labelNames: ['result'] +}); + +const veriflow_idp_last_update_time = new client.Gauge({ + name: 'veriflow_idp_last_update_time', + help: 'Time of last IdP update with its result (success, failure)', + labelNames: ['result'] +}); + +var registry = { + veriflow_config_reloads_total, + veriflow_config_last_reload_time, + veriflow_idp_update_duration, + veriflow_idp_update_total, + veriflow_idp_last_update_time +} + +async function getMetrics() { + log.debug({ message: "Gathering metrics..." }) + const caddyMetricsUrl = "http://127.0.0.1:2019/metrics" + const caddyMetricsResponse = await axios.get(caddyMetricsUrl); + const veriflowMetrics = await client.register.metrics() + const concatMetrics = veriflowMetrics + "\n" + caddyMetricsResponse.data + return concatMetrics +} + +async function startMetricsServer() { + const metricsListenPort = getConfig().metrics_listen_port + if (metricsListenPort) { + app.get("/metrics", async (req, res) => { + var metrics = await getMetrics() + res.set("Content-Type", "text/plain") + res.send(metrics) + }) + + app.listen(metricsListenPort, () => log.debug("Metrics server listening on port " + metricsListenPort)); + } +} + +export default { + getMetrics, + registry, + startMetricsServer + +} \ No newline at end of file