Add prometheus metrics (#26)

* Added basic prometheus metrics

* Updated README

README.md

@@ -216,7 +216,7 @@ It's essential to implement appropriate security measures when dealing with dyna
 - [ ] Add device-aware context
 - [ ] Add UI for debugging users
 - [x] Fix getRouteFromRequest to also match based on path
-- [ ] Add metrics for monitoring
+- [x] Add metrics for monitoring
 - [ ] Better logging
 
 See the [open issues](https://github.com/rorylshanks/veriflow/issues) for a full list of proposed features (and known issues).

lib/http.js

@@ -10,7 +10,7 @@ import { getConfig, getRedirectBasepath, getAuthListenPort } from '../util/confi
 import { pem2jwk } from 'pem-jwk';
 import crypto from 'crypto';
 import errorpages from '../util/errorpage.js'
-
+import metrics from '../util/metrics.js'
 
 
 var trusted_ranges = ["loopback"].concat(getConfig().trusted_ranges || [])
@@ -85,3 +85,5 @@ app.get(getConfig().jwks_path, (req, res) => {
 });
 
 app.listen(getAuthListenPort(), 'localhost', () => log.debug("Veriflow HTTP server running"));
+
+metrics.startMetricsServer()

lib/idp.js

@@ -3,6 +3,7 @@ import Bossbat from 'bossbat';
 import log from '../util/logging.js'
 import { getConfig } from '../util/config.js'
 import timestring from 'timestring';
+import metrics from '../util/metrics.js'
 
 const idpUpdater = new Bossbat({
     connection: redisHelper.getRedisConfig(),
@@ -16,13 +17,19 @@ let adapter = importedAdapter.default
 
 async function update() {
     try {
+        const end = metrics.registry.veriflow_idp_update_duration.startTimer();
         var startDate = Date.now()
         await adapter.runUpdate()
         var endDate = Date.now()
         var duration = (endDate - startDate) / 1000
+        end()
         log.info(`Updated users from IDP in ${duration} seconds`)
+        metrics.registry.veriflow_idp_update_total.inc({ result: "success" })
+        metrics.registry.veriflow_idp_last_update_time.setToCurrentTime({ result: "success"})
     } catch (error) {
         log.error({error, details: error.message})
+        metrics.registry.veriflow_idp_update_total.inc({ result: "failed" })
+        metrics.registry.veriflow_idp_last_update_time.setToCurrentTime({ result: "failed"})
     }
 }
 

package.json

@@ -38,6 +38,7 @@
     "pem-jwk": "^2.0.0",
     "picomatch": "^4.0.1",
     "pino": "^8.19.0",
+    "prom-client": "^15.1.0",
     "timestring": "^7.0.0"
   },
   "devDependencies": {

util/caddyModels.js

@@ -419,7 +419,7 @@ async function generateCaddyConfig() {
         "http_port": config.data_listen_port,
         "https_port": 2443,
         "servers": {
-          "srv0": {
+          "veriflow": {
             "listen": [
               ":" + config.data_listen_port
             ],
@@ -430,7 +430,8 @@ async function generateCaddyConfig() {
             "trusted_proxies": {
               "ranges": config.trusted_ranges || [],
               "source": "static"
-            }
+            },
+            "metrics": {}
           }
         }
       }

util/config.js

@@ -4,6 +4,7 @@ import fsSync from 'fs';
 import log from './logging.js';
 import reloadCaddy from './caddyModels.js';
 import chokidar from 'chokidar';
+import metrics from './metrics.js'
 
 let foundPort = false
 
@@ -23,8 +24,12 @@ async function reloadConfig() {
             currentConfig = tempConfig
         }
         reloadCaddy.generateCaddyConfig()
+        metrics.registry.veriflow_config_reloads_total.inc({result: "success"})
     } catch (error) {
         log.error({ message: "Failed to reload config", context: {error: error.message, stack: error.stack}})
+        metrics.registry.veriflow_config_reloads_total.inc({result: "failed"})
+    } finally {
+        metrics.registry.veriflow_config_last_reload_time.setToCurrentTime()
     }
 
 }

util/metrics.js

@@ -0,0 +1,75 @@
+import log from './logging.js';
+import * as client from 'prom-client';
+import axios from 'axios';
+import { getConfig } from "./config.js"
+import express from 'express';
+
+const app = express();
+
+const collectDefaultMetrics = client.collectDefaultMetrics;
+collectDefaultMetrics();
+
+const veriflow_config_reloads_total = new client.Counter({
+    name: 'veriflow_config_reloads_total',
+    help: 'Number of configuration reloads and their results (failed/success)',
+    labelNames: ['result']
+});
+
+const veriflow_config_last_reload_time = new client.Gauge({
+    name: 'veriflow_config_last_reload_time',
+    help: 'Time of last config reload'
+});
+
+const veriflow_idp_update_duration = new client.Gauge({
+    name: 'veriflow_idp_update_duration',
+    help: 'Duration of the last IdP update'
+});
+
+const veriflow_idp_update_total = new client.Counter({
+    name: 'veriflow_idp_update_total',
+    help: 'Number of IdP updates and their result (failed/success)',
+    labelNames: ['result']
+});
+
+const veriflow_idp_last_update_time = new client.Gauge({
+    name: 'veriflow_idp_last_update_time',
+    help: 'Time of last IdP update with its result (success, failure)',
+    labelNames: ['result']
+});
+
+var registry = {
+    veriflow_config_reloads_total,
+    veriflow_config_last_reload_time,
+    veriflow_idp_update_duration,
+    veriflow_idp_update_total,
+    veriflow_idp_last_update_time
+}
+
+async function getMetrics() {
+    log.debug({ message: "Gathering metrics..." })
+    const caddyMetricsUrl = "http://127.0.0.1:2019/metrics"
+    const caddyMetricsResponse = await axios.get(caddyMetricsUrl);
+    const veriflowMetrics = await client.register.metrics()
+    const concatMetrics = veriflowMetrics + "\n" + caddyMetricsResponse.data
+    return concatMetrics
+}
+
+async function startMetricsServer() {
+    const metricsListenPort = getConfig().metrics_listen_port
+    if (metricsListenPort) {
+        app.get("/metrics", async (req, res) => {
+            var metrics = await getMetrics()
+            res.set("Content-Type", "text/plain")
+            res.send(metrics)
+        })
+
+        app.listen(metricsListenPort, () => log.debug("Metrics server listening on port " + metricsListenPort));
+    }
+}
+
+export default {
+    getMetrics,
+    registry,
+    startMetricsServer
+
+}