From d83754549351d5568ffb529ed223f14867856986 Mon Sep 17 00:00:00 2001 From: tom goriunov Date: Thu, 4 Apr 2024 12:49:21 +0200 Subject: [PATCH] Monitoring: social preview bots request counter (#1781) * add counter metric and logger for bot request * add script for grafana local instance * add log endpoint * add ENV to enable monitoring * [skip ci] change metric labels and add telegram bot * [skip ci] remove 404 from /metrics route --- .gitignore | 1 + deploy/values/review/values.yaml.gotmpl | 1 + lib/metadata/getApiDataForSocialPreview.ts | 32 ++++++++++++++++++++++ lib/metadata/getPageOgType.ts | 2 ++ lib/metadata/templates/description.ts | 2 ++ lib/metadata/templates/title.ts | 2 ++ lib/mixpanel/getPageType.ts | 2 ++ lib/monitoring/metrics.ts | 20 ++++++++++++++ nextjs/nextjs-routes.d.ts | 2 ++ package.json | 5 +++- pages/_document.tsx | 3 ++ pages/api/log.ts | 9 ++++++ pages/api/metrics.ts | 13 +++++++++ prometheus.yml | 16 +++++++++++ yarn.lock | 25 +++++++++++++++++ 15 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 lib/metadata/getApiDataForSocialPreview.ts create mode 100644 lib/monitoring/metrics.ts create mode 100644 pages/api/log.ts create mode 100644 pages/api/metrics.ts create mode 100644 prometheus.yml diff --git a/.gitignore b/.gitignore index c877d7bbea..721584d133 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ .DS_Store *.pem .tools +grafana # debug npm-debug.log* diff --git a/deploy/values/review/values.yaml.gotmpl b/deploy/values/review/values.yaml.gotmpl index 7f6418f43f..91313e4b12 100644 --- a/deploy/values/review/values.yaml.gotmpl +++ b/deploy/values/review/values.yaml.gotmpl @@ -90,6 +90,7 @@ frontend: NEXT_PUBLIC_AD_ADBUTLER_CONFIG_DESKTOP: "{ \"id\": \"632019\", \"width\": \"728\", \"height\": \"90\" }" NEXT_PUBLIC_AD_ADBUTLER_CONFIG_MOBILE: "{ \"id\": \"632018\", \"width\": \"320\", \"height\": \"100\" }" NEXT_PUBLIC_DATA_AVAILABILITY_ENABLED: true + PROMETHEUS_METRICS_ENABLED: true envFromSecret: NEXT_PUBLIC_SENTRY_DSN: ref+vault://deployment-values/blockscout/dev/review?token_env=VAULT_TOKEN&address=https://vault.k8s.blockscout.com#/NEXT_PUBLIC_SENTRY_DSN SENTRY_CSP_REPORT_URI: ref+vault://deployment-values/blockscout/dev/review?token_env=VAULT_TOKEN&address=https://vault.k8s.blockscout.com#/SENTRY_CSP_REPORT_URI diff --git a/lib/metadata/getApiDataForSocialPreview.ts b/lib/metadata/getApiDataForSocialPreview.ts new file mode 100644 index 0000000000..8357d1e2ef --- /dev/null +++ b/lib/metadata/getApiDataForSocialPreview.ts @@ -0,0 +1,32 @@ +import type { IncomingMessage, ServerResponse } from 'http'; + +import { httpLogger } from 'nextjs/utils/logger'; + +import metrics from 'lib/monitoring/metrics'; + +export default async function getApiDataForSocialPreview(req: IncomingMessage | undefined, res: ServerResponse | undefined, pathname: string) { + if (!req || !res || !metrics) { + return; + } + + const userAgent = req.headers['user-agent']; + + if (!userAgent) { + return; + } + + if (userAgent.toLowerCase().includes('twitter')) { + httpLogger(req, res); + metrics.requestCounter.inc({ route: pathname, bot: 'twitter' }); + } + + if (userAgent.toLowerCase().includes('facebook')) { + httpLogger(req, res); + metrics.requestCounter.inc({ route: pathname, bot: 'facebook' }); + } + + if (userAgent.toLowerCase().includes('telegram')) { + httpLogger(req, res); + metrics.requestCounter.inc({ route: pathname, bot: 'telegram' }); + } +} diff --git a/lib/metadata/getPageOgType.ts b/lib/metadata/getPageOgType.ts index ef7ef24e26..a195848744 100644 --- a/lib/metadata/getPageOgType.ts +++ b/lib/metadata/getPageOgType.ts @@ -48,6 +48,8 @@ const OG_TYPE_DICT: Record = { // service routes, added only to make typescript happy '/login': 'Regular page', + '/api/metrics': 'Regular page', + '/api/log': 'Regular page', '/api/media-type': 'Regular page', '/api/proxy': 'Regular page', '/api/csrf': 'Regular page', diff --git a/lib/metadata/templates/description.ts b/lib/metadata/templates/description.ts index c064709b6f..dbaaf5e0fc 100644 --- a/lib/metadata/templates/description.ts +++ b/lib/metadata/templates/description.ts @@ -51,6 +51,8 @@ const TEMPLATE_MAP: Record = { // service routes, added only to make typescript happy '/login': DEFAULT_TEMPLATE, + '/api/metrics': DEFAULT_TEMPLATE, + '/api/log': DEFAULT_TEMPLATE, '/api/media-type': DEFAULT_TEMPLATE, '/api/proxy': DEFAULT_TEMPLATE, '/api/csrf': DEFAULT_TEMPLATE, diff --git a/lib/metadata/templates/title.ts b/lib/metadata/templates/title.ts index 9a667bb9ce..acf0c4f87e 100644 --- a/lib/metadata/templates/title.ts +++ b/lib/metadata/templates/title.ts @@ -46,6 +46,8 @@ const TEMPLATE_MAP: Record = { // service routes, added only to make typescript happy '/login': 'login', + '/api/metrics': 'node API prometheus metrics', + '/api/log': 'node API request log', '/api/media-type': 'node API media type', '/api/proxy': 'node API proxy', '/api/csrf': 'node API CSRF token', diff --git a/lib/mixpanel/getPageType.ts b/lib/mixpanel/getPageType.ts index fb4c48e7fe..f74a16fb51 100644 --- a/lib/mixpanel/getPageType.ts +++ b/lib/mixpanel/getPageType.ts @@ -46,6 +46,8 @@ export const PAGE_TYPE_DICT: Record = { // service routes, added only to make typescript happy '/login': 'Login', + '/api/metrics': 'Node API: Prometheus metrics', + '/api/log': 'Node API: Request log', '/api/media-type': 'Node API: Media type', '/api/proxy': 'Node API: Proxy', '/api/csrf': 'Node API: CSRF token', diff --git a/lib/monitoring/metrics.ts b/lib/monitoring/metrics.ts new file mode 100644 index 0000000000..db722561ce --- /dev/null +++ b/lib/monitoring/metrics.ts @@ -0,0 +1,20 @@ +import * as promClient from 'prom-client'; + +const metrics = (() => { + // eslint-disable-next-line no-restricted-properties + if (process.env.PROMETHEUS_METRICS_ENABLED !== 'true') { + return; + } + + promClient.register.clear(); + + const requestCounter = new promClient.Counter({ + name: 'request_counter', + help: 'Number of incoming requests', + labelNames: [ 'route', 'bot' ] as const, + }); + + return { requestCounter }; +})(); + +export default metrics; diff --git a/nextjs/nextjs-routes.d.ts b/nextjs/nextjs-routes.d.ts index ee87eb5dc1..5f5a8edd27 100644 --- a/nextjs/nextjs-routes.d.ts +++ b/nextjs/nextjs-routes.d.ts @@ -18,7 +18,9 @@ declare module "nextjs-routes" { | DynamicRoute<"/address/[hash]", { "hash": string }> | StaticRoute<"/api/csrf"> | StaticRoute<"/api/healthz"> + | StaticRoute<"/api/log"> | StaticRoute<"/api/media-type"> + | StaticRoute<"/api/metrics"> | StaticRoute<"/api/proxy"> | StaticRoute<"/api-docs"> | DynamicRoute<"/apps/[id]", { "id": string }> diff --git a/package.json b/package.json index 96e564d871..3c85287b18 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,9 @@ "test:pw:detect-affected": "node ./deploy/tools/affected-tests/index.js", "test:jest": "jest", "test:jest:watch": "jest --watch", - "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh" + "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh", + "monitoring:prometheus:local": "docker run --name blockscout_prometheus -d -p 127.0.0.1:9090:9090 -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus", + "monitoring:grafana:local": "docker run -d -p 4000:3000 --name=blockscout_grafana --user $(id -u) --volume $(pwd)/grafana:/var/lib/grafana grafana/grafana-enterprise" }, "dependencies": { "@chakra-ui/react": "2.7.1", @@ -87,6 +89,7 @@ "phoenix": "^1.6.15", "pino-http": "^8.2.1", "pino-pretty": "^9.1.1", + "prom-client": "15.1.1", "qrcode": "^1.5.1", "react": "18.2.0", "react-device-detect": "^2.2.3", diff --git a/pages/_document.tsx b/pages/_document.tsx index 74ebe08fb5..2101c04c4f 100644 --- a/pages/_document.tsx +++ b/pages/_document.tsx @@ -5,6 +5,7 @@ import React from 'react'; import * as serverTiming from 'nextjs/utils/serverTiming'; +import getApiDataForSocialPreview from 'lib/metadata/getApiDataForSocialPreview'; import theme from 'theme'; import * as svgSprite from 'ui/shared/IconSvg'; @@ -21,6 +22,8 @@ class MyDocument extends Document { return result; }; + await getApiDataForSocialPreview(ctx.req, ctx.res, ctx.pathname); + const initialProps = await Document.getInitialProps(ctx); return initialProps; diff --git a/pages/api/log.ts b/pages/api/log.ts new file mode 100644 index 0000000000..cb89b35f59 --- /dev/null +++ b/pages/api/log.ts @@ -0,0 +1,9 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; + +import { httpLogger } from 'nextjs/utils/logger'; + +export default async function logHandler(req: NextApiRequest, res: NextApiResponse) { + httpLogger(req, res); + + res.status(200).send('ok'); +} diff --git a/pages/api/metrics.ts b/pages/api/metrics.ts new file mode 100644 index 0000000000..d2f00ff430 --- /dev/null +++ b/pages/api/metrics.ts @@ -0,0 +1,13 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import * as promClient from 'prom-client'; + +// eslint-disable-next-line no-restricted-properties +const isEnabled = process.env.PROMETHEUS_METRICS_ENABLED === 'true'; + +isEnabled && promClient.collectDefaultMetrics(); + +export default async function metricsHandler(req: NextApiRequest, res: NextApiResponse) { + const metrics = await promClient.register.metrics(); + res.setHeader('Content-type', promClient.register.contentType); + res.send(metrics); +} diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000000..89b95e0ecb --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + # - "first.rules" + # - "second.rules" + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + - job_name: frontend + metrics_path: /node-api/metrics + static_configs: + - targets: ['host.docker.internal:3000'] \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 0e2880182d..353e3c618c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3528,6 +3528,11 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.7.0.tgz#b139c81999c23e3c8d3c0a7234480e945920fc40" integrity sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw== +"@opentelemetry/api@^1.4.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" + integrity sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w== + "@opentelemetry/auto-instrumentations-node@^0.39.4": version "0.39.4" resolved "https://registry.yarnpkg.com/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.39.4.tgz#c772201ba224e6ebffaaf17c9f2234e6e8343328" @@ -7726,6 +7731,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + bl@^4.0.3: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -13300,6 +13310,14 @@ progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +prom-client@15.1.1: + version "15.1.1" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.1.tgz#71ba84371241acd173181b04a436782c246f3652" + integrity sha512-GVA2H96QCg2q71rjc3VYvSrVG7OpnJxyryC7dMzvfJfpJJHzQVwF3TJLfHzKORcwJpElWs1TwXLthlJAFJxq2A== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + promise-polyfill@^8.1.3: version "8.3.0" resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.3.0.tgz#9284810268138d103807b11f4e23d5e945a4db63" @@ -14901,6 +14919,13 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + test-exclude@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e"