addes environment variables

- added environment variable `API_GLOBAL_LIMIT_WINDOW_MS` - added environment variable `API_GLOBAL_LIMIT_MAX` - added environment variable `API_TELEMETRY_LIMIT_WINDOW_MS` - added environment variable `API_TELEMETRY_LIMIT_MAX` - added environment variable `API_TELEMETRY_LIMIT_MAX` - enrich log with rate limit data - added .env file support
pixelit-project · Jul 27, 2023 · f5446c1 · f5446c1
1 parent fbbba87
commit f5446c1
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,9 @@
 
 This is the API service for the [PixelIt](https://github.com/pixelit-project/PixelIt) project, this provides the bitmaps for the Pixelit.
 
+[![CodeQL](https://github.com/pixelit-project/PixelIt.API/actions/workflows/codeql.yml/badge.svg)](https://github.com/pixelit-project/PixelIt.API/actions/workflows/codeql.yml)
+
+_____
 <a href="https://t.me/pixelitdisplay">
     <img src="https://img.shields.io/endpoint?label=Telegram&style=for-the-badge&url=https%3A%2F%2Frunkit.io%2Fdamiankrawczyk%2Ftelegram-badge%2Fbranches%2Fmaster%3Furl%3Dhttps%3A%2F%2Ft.me%2Fpixelitdisplay"/>
 </a> 
@@ -30,6 +33,10 @@ pixelit_api:
     MYSQL_USER: user
     MYSQL_PASSWORD: password
     GITHUB_TOKEN: token
+    API_GLOBAL_LIMIT_WINDOW_MS: 300000
+    API_GLOBAL_LIMIT_MAX: 100
+    API_TELEMETRY_LIMIT_WINDOW_MS: 900000
+    API_TELEMETRY_LIMIT_MAX: 10
     SEQ_SERVER: http://seqserver:5341
     SEQ_APIKEY: xxxxxxxxxxxx
 ```
@@ -40,6 +47,16 @@ Install dependencies with `npm install` and run dev server with `npn run dev`.
 
 ## Changelog
 
+### 1.2.0 (2023-07-27)
+
+- (o0shojo0o) added environment variable `API_GLOBAL_LIMIT_WINDOW_MS`
+- (o0shojo0o) added environment variable `API_GLOBAL_LIMIT_MAX`
+- (o0shojo0o) added environment variable `API_TELEMETRY_LIMIT_WINDOW_MS`
+- (o0shojo0o) added environment variable `API_TELEMETRY_LIMIT_MAX`
+- (o0shojo0o) added environment variable `API_TELEMETRY_LIMIT_MAX`
+- (o0shojo0o) enrich log with rate limit data
+- (o0shojo0o) added .env file support 
+
 ### 1.1.1 (2023-02-14)
 
 - (o0shojo0o) added rate limits

diff --git a/dockerfile b/dockerfile
@@ -1,4 +1,4 @@
-FROM node:16
+FROM node:18
 
 # Create app directory
 WORKDIR /usr/src/app

diff --git a/libs/logger.js b/libs/logger.js
@@ -34,6 +34,9 @@ function prepareLogForConsole(message, properties, dateTime) {
         if (properties.useragent) {
             extendedInformation += `, UserAgent: ${properties.useragent.browser}`
         }
+        if (properties.rateLimit) {
+            extendedInformation += `, RateLimit: ${properties.rateLimit.current} of ${properties.rateLimit.limit} (${properties.rateLimit.remaining}) reset in ${(new Date(properties.rateLimit.resetTime).getTime() - Date.now()) / 1000}s`
+        }
         extendedInformation += ']'
     }
 
@@ -42,12 +45,7 @@ function prepareLogForConsole(message, properties, dateTime) {
             message = message.replace(`{${key}}`, properties[key])
         }
     }
-    return `[${dateTime
-        .toISOString()
-        .slice(
-            0,
-            10
-        )}T${dateTime.toLocaleTimeString()}]${extendedInformation} ${message}`
+    return `[${dateTime.toISOString().slice(0, 10)}T${dateTime.toLocaleTimeString()}]${extendedInformation} ${message}`
 }
 
 module.exports = {

diff --git a/main.js b/main.js
@@ -1,3 +1,4 @@
+require('dotenv').config()
 const express = require('express');
 const useragent = require('express-useragent');
 const rateLimit = require('express-rate-limit')
@@ -24,8 +25,8 @@ app.use(bodyParser.json());
 app.use(cors());
 
 const apiLimiter = rateLimit({
-    windowMs: 5 * 60 * 1000, // 5 minutes
-    max: 100,
+    windowMs: Number(process.env.API_GLOBAL_LIMIT_WINDOW_MS) || 5 * 60 * 1000, // 5 minutes
+    max: Number(process.env.API_GLOBAL_LIMIT_MAX) || 100,
     standardHeaders: true,
     legacyHeaders: false,
     onLimitReached: (req, response, next, options) => {
@@ -37,8 +38,8 @@ const apiLimiter = rateLimit({
 });
 
 const telemetryLimiter = rateLimit({
-    windowMs: 15 * 60 * 1000, // 15 minutes
-    max: 10,
+    windowMs: Number(process.env.API_TELEMETRY_LIMIT_WINDOW_MS) || 15 * 60 * 1000, // 15 minutes
+    max: Number(process.env.API_TELEMETRY_LIMIT_MAX) || 10,
     standardHeaders: true,
     legacyHeaders: false,
     onLimitReached: (req, response, next, options) => {
@@ -56,22 +57,22 @@ app.get('/api/GetBMPByID/:id', async (req, res) => {
     const id = req.params.id;
 
     if (tools.isNumeric(id) == false) {
-        log.warn('GetBMPByID: {id} is not a valid ID!', { id, sourceIP, rawUrl, useragent: req.useragent, });
+        log.warn('{apiPath}: {id} is not a valid ID!', { apiPath: 'GetBMPByID', id, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
         res.status(400).send('Not valid ID');
         return;
     }
 
     const bmp = (await cache.getOrSet(`GetBMPByID_${id}`, () => { return repo.getBMPByID(id) }, 0)) ?? {};
 
-    log.info('GetBMPByID: BMP with ID {id} and name {name} successfully delivered', { id: bmp.id, name: bmp.name, sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: BMP with ID {id} and name {name} successfully delivered', { apiPath: 'GetBMPByID', id: bmp.id, name: bmp.name, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
     res.send(bmp);
 });
 
 app.get('/api/GetBMPNewst', async (req, res) => {
     const sourceIP = tools.getIPFromRequest(req);
     const rawUrl = tools.getRawURLFromRequest(req);
     const bmp = (await cache.getOrSet('GetBMPNewst', () => { return repo.getBMPNewst() }, 30)) ?? {};
-
+    log.info('{apiPath} BMP with ID {id} and name {name} successfully delivered', { apiPath: 'GetBMPNewst', id: bmp.id, name: bmp.name, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
     res.send(bmp);
 });
 
@@ -80,7 +81,7 @@ app.get('/api/GetBMPAll', async (req, res) => {
     const rawUrl = tools.getRawURLFromRequest(req);
     const bmps = (await cache.getOrSet('GetBMPAll', () => { return repo.getBMPAll() }, 30)) ?? [];
 
-    log.info('GetBMPAll: {count} BMPs successfully delivered', { count: bmps.length, sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: {count} BMPs successfully delivered', { apiPath: 'GetBMPAll', count: bmps.length, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
     res.send(bmps);
 });
 
@@ -89,14 +90,14 @@ app.post('/api/Telemetry', telemetryLimiter, async (req, res) => {
     const rawUrl = tools.getRawURLFromRequest(req);
 
     if (!req.body) {
-        log.error('Telemetry: No body found', { sourceIP, rawUrl, useragent: req.useragent, });
+        log.error('{apiPath}: No body found', { apiPath: 'Telemetry', sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
         res.status(400).send('Not valid body');
         return;
     }
 
     (async () => {
         req.body.geoip = await geoip.lookup(sourceIP);
-        log.info(`Telemetry: ${JSON.stringify(req.body)}`, { sourceIP, rawUrl, useragent: req.useragent, });
+        log.info(`{apiPath}: ${JSON.stringify(req.body)}`, { apiPath: 'Telemetry', sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
         repo.saveStats(req.body);
     })();
 
@@ -108,7 +109,7 @@ app.get('/api/UserMapData', async (req, res) => {
     const rawUrl = tools.getRawURLFromRequest(req);
     const userMapData = (await cache.getOrSet('UserMapData', () => { return repo.getUserMapData() }, 30)) ?? [];
 
-    log.info('UserMapData: {count} User successfully delivered', { count: userMapData.length, sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: {count} User successfully delivered', { apiPath: 'UserMapData', count: userMapData.length, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
     res.send(userMapData);
 });
 
@@ -126,7 +127,7 @@ app.get('/api/LastVersion', async (req, res) => {
         delete lastReleaseData[key];
     }
 
-    log.info('LastVersion: Version {version} successfully delivered', { version: lastReleaseData.version, sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: Version {version} successfully delivered', { apiPath: 'LastVersion', version: lastReleaseData.version, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
 
     res.send(lastReleaseData);
 });
@@ -141,7 +142,7 @@ app.get('/api/LastRelease', async (req, res) => {
         lastReleaseData = releases[0];
     }
 
-    log.info('LastRelease: Version {version} successfully delivered', { version: lastReleaseData.version, sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: Version {version} successfully delivered', { apiPath: 'LastRelease', version: lastReleaseData.version, sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
 
 
     res.send(lastReleaseData);
@@ -152,7 +153,7 @@ app.get('/api/Releases', async (req, res) => {
     const rawUrl = tools.getRawURLFromRequest(req);
     const releases = await cache.getOrSet('Releases', () => { return gitRepo.getGitReleases() }, 600) ?? [];
 
-    log.info('Releases: Versions {versions} successfully delivered', { versions: releases.map(value => value.version).join(', '), sourceIP, rawUrl, useragent: req.useragent, });
+    log.info('{apiPath}: Versions {versions} successfully delivered', { apiPath: 'Releases', versions: releases.map(value => value.version).join(', '), sourceIP, rawUrl, useragent: req.useragent, rateLimit: req.rateLimit, });
 
     res.send(releases);
 });

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -21,6 +21,7 @@
     "axios": "^1.4.0",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",
+    "dotenv": "^16.3.1",
     "express": "^4.18.2",
     "express-rate-limit": "^6.8.1",
     "express-useragent": "^1.0.15",