From 52d81bc1fff0b1d7c3d7665b2ef453e77a4996de Mon Sep 17 00:00:00 2001 From: Isomer Admin <60170571+isomeradmin@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:40:59 +0800 Subject: [PATCH 1/7] fix: package.json & package-lock.json to reduce vulnerabilities (#1414) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-WS-7266574 Co-authored-by: snyk-bot --- package-lock.json | 261 ++++++++++++++++++++++++++++++++++++++-------- package.json | 2 +- 2 files changed, 220 insertions(+), 43 deletions(-) diff --git a/package-lock.json b/package-lock.json index f0b754690..cc4191857 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,7 @@ "isomorphic-git": "^1.25.10", "joi": "^17.13.1", "js-base64": "^3.7.7", - "jsdom": "^16.7.0", + "jsdom": "^17.0.0", "jsonwebtoken": "^9.0.0", "lodash": "^4.17.21", "marked": "^12.0.2", @@ -6591,7 +6591,8 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", - "deprecated": "Use your platform's native atob() and btoa() methods instead" + "deprecated": "Use your platform's native atob() and btoa() methods instead", + "license": "BSD-3-Clause" }, "node_modules/abbrev": { "version": "1.1.1", @@ -9200,9 +9201,10 @@ "integrity": "sha512-/RC5F4l1SCqD/jazwUF6+t34Cd8zTSAGZ7rvvZu1whZUhD2a5MOGKjSGowoGcpj/cbVZk1ZODIooJEQQq3nNAA==" }, "node_modules/cssom": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", - "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" }, "node_modules/cssstyle": { "version": "2.3.0", @@ -9312,16 +9314,60 @@ } }, "node_modules/data-urls": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", - "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", + "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==", + "license": "MIT", "dependencies": { - "abab": "^2.0.3", - "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.0.0" + "abab": "^2.0.6", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^11.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "license": "MIT", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "license": "MIT", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" } }, "node_modules/data-view-buffer": { @@ -13348,6 +13394,125 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/jest-environment-jsdom/node_modules/acorn": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.0.tgz", + "integrity": "sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/jest-environment-jsdom/node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-environment-jsdom/node_modules/data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-environment-jsdom/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-environment-jsdom/node_modules/jsdom": { + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "abab": "^2.0.5", + "acorn": "^8.2.4", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "form-data": "^3.0.0", + "html-encoding-sniffer": "^2.0.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.5.0", + "ws": "^7.4.6", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jest-environment-jsdom/node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-environment-jsdom/node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/jest-environment-node": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-27.5.1.tgz", @@ -15124,20 +15289,21 @@ } }, "node_modules/jsdom": { - "version": "16.7.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", - "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz", + "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==", + "license": "MIT", "dependencies": { "abab": "^2.0.5", - "acorn": "^8.2.4", + "acorn": "^8.4.1", "acorn-globals": "^6.0.0", - "cssom": "^0.4.4", + "cssom": "^0.5.0", "cssstyle": "^2.3.0", - "data-urls": "^2.0.0", - "decimal.js": "^10.2.1", + "data-urls": "^3.0.0", + "decimal.js": "^10.3.1", "domexception": "^2.0.1", "escodegen": "^2.0.0", - "form-data": "^3.0.0", + "form-data": "^4.0.0", "html-encoding-sniffer": "^2.0.1", "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", @@ -15152,12 +15318,12 @@ "webidl-conversions": "^6.1.0", "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", - "whatwg-url": "^8.5.0", - "ws": "^7.4.6", + "whatwg-url": "^9.0.0", + "ws": "^8.0.0", "xml-name-validator": "^3.0.0" }, "engines": { - "node": ">=10" + "node": ">=12" }, "peerDependencies": { "canvas": "^2.5.0" @@ -15179,24 +15345,32 @@ "node": ">=0.4.0" } }, - "node_modules/jsdom/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/jsdom/node_modules/parse5": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" }, + "node_modules/jsdom/node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -19789,6 +19963,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "license": "MIT", "dependencies": { "punycode": "^2.1.1" }, @@ -20594,6 +20769,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "license": "BSD-2-Clause", "engines": { "node": ">=10.4" } @@ -20609,19 +20785,20 @@ "node_modules/whatwg-mimetype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", + "license": "MIT" }, "node_modules/whatwg-url": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", - "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", + "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", + "license": "MIT", "dependencies": { - "lodash": "^4.7.0", "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" }, "engines": { - "node": ">=10" + "node": ">=12" } }, "node_modules/which": { diff --git a/package.json b/package.json index 58648b9b8..e94e27499 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "isomorphic-git": "^1.25.10", "joi": "^17.13.1", "js-base64": "^3.7.7", - "jsdom": "^16.7.0", + "jsdom": "^17.0.0", "jsonwebtoken": "^9.0.0", "lodash": "^4.17.21", "marked": "^12.0.2", From b37c3db213a26e4470083fd547b74cd3b9c8aad0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 07:46:47 +0000 Subject: [PATCH 2/7] chore(deps): bump braces from 3.0.2 to 3.0.3 (#1413) Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3. - [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/braces/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: braces dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index cc4191857..c24445a96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7409,11 +7409,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -11114,9 +11114,9 @@ "devOptional": true }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, From e2a085873027c3a50b2318f3405530ac1f8b7b6c Mon Sep 17 00:00:00 2001 From: Harish Date: Wed, 26 Jun 2024 16:45:04 +0800 Subject: [PATCH 3/7] chore: update slack-bolt (#1415) Co-authored-by: Alexander Lee --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index c24445a96..d8dcd7f06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@octokit/rest": "^18.12.0", "@opengovsg/formsg-sdk": "^0.11.0", "@opengovsg/sgid-client": "^2.0.0", - "@slack/bolt": "^3.18.0", + "@slack/bolt": "^3.19.0", "auto-bind": "^4.0.0", "aws-lambda": "^1.0.7", "aws-sdk": "^2.1619.0", @@ -5028,9 +5028,9 @@ } }, "node_modules/@slack/bolt": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@slack/bolt/-/bolt-3.18.0.tgz", - "integrity": "sha512-A7bDi5kY50fS6/nsmURkQdO3iMxD8aX/rA+m1UXEM2ue2z4KijeQtx2sOZ4YkJQ/h7BsgTQM0CYh3qqmo+m5sQ==", + "version": "3.19.0", + "resolved": "https://registry.npmjs.org/@slack/bolt/-/bolt-3.19.0.tgz", + "integrity": "sha512-P5Yup/PbO8sE5xsuqkBkpSPkxEkfWZ6yo5ZlmBGxRhhoU1usUSU2w0bgZoiDX4WFm7ZX+3x2Dyf4VMa9kzfmVQ==", "dependencies": { "@slack/logger": "^4.0.0", "@slack/oauth": "^2.6.2", diff --git a/package.json b/package.json index e94e27499..c655eae21 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "@octokit/rest": "^18.12.0", "@opengovsg/formsg-sdk": "^0.11.0", "@opengovsg/sgid-client": "^2.0.0", - "@slack/bolt": "^3.18.0", + "@slack/bolt": "^3.19.0", "auto-bind": "^4.0.0", "aws-lambda": "^1.0.7", "aws-sdk": "^2.1619.0", From 8c47822f4fd33dd32c2c17ff515b3a0bda16b504 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Thu, 27 Jun 2024 14:08:52 +0800 Subject: [PATCH 4/7] feat(monitoring): add dns reporter (#1376) ## Problem This is a first pr that is up to add some level of sane reporting. While scheduling is part of this feature, it is not within the scope of this pr. This pr only adds (currently dead code) logic to grab the domains that we own in isomer, and do a dns dig. This is meant to be verbose, and in the future alarms can be added based on the results of this. This is not meant to replace monitoring, it is just meant to fine tune some blind spots that uptime robot currently has + some sane checker during incident response to show history of dns records for a site that we manage. I am opting to log it directly in our backend to keep things simple. will add alarms + the scheduler in subsequent prs. ## Solution grab ALL domains from keycdn + amplify + redirection records + log dns records on them. **Breaking Changes** - [ ] Yes - this PR contains breaking changes - Details ... - [X] No - this PR is backwards compatible with ALL of the following feature flags in this [doc](https://www.notion.so/opengov/Existing-feature-flags-518ad2cdc325420893a105e88c432be5) ## Tests in server.ts add: `monitoringService.driver()` should see this in the logs: ![Screenshot 2024-05-15 at 5.48.05 PM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/4JosFH65rhzwIvkZw2J6/2bf61e7f-0ec4-466f-87b7-ec7e1d84993e.png) ## Deploy Notes **New environment variables**: - `KEYCDN_API_KEY` : to get all the zones that we own in keycdn - `S3_BUCKET_NAME`: bucket name - [ ] HAVE NOT added env var to 1PW + SSM script (`fetch_ssm_parameters.sh`) **New scripts**: - `script` : script details **New dependencies**: - `dependency` : dependency details **New dev dependencies**: - `dependency` : dependency details --- .aws/deploy/support-task-definition.prod.json | 1 + .../support-task-definition.staging.json | 1 + .env.test | 2 + common/index.ts | 5 + package-lock.json | 181 +++++-------- package.json | 2 +- src/config/config.ts | 9 + src/errors/MonitoringError.ts | 11 + src/monitoring/index.ts | 244 ++++++++++++++++++ src/server.ts | 1 + src/services/identity/LaunchesService.ts | 6 + src/services/identity/ReposService.ts | 2 +- src/utils/papa-parse.ts | 19 ++ support/index.ts | 10 +- 14 files changed, 377 insertions(+), 117 deletions(-) create mode 100644 src/errors/MonitoringError.ts create mode 100644 src/monitoring/index.ts create mode 100644 src/utils/papa-parse.ts diff --git a/.aws/deploy/support-task-definition.prod.json b/.aws/deploy/support-task-definition.prod.json index 97ed65ab2..7db06bc19 100644 --- a/.aws/deploy/support-task-definition.prod.json +++ b/.aws/deploy/support-task-definition.prod.json @@ -98,6 +98,7 @@ "valueFrom": "PROD_ISOMERPAGES_REPO_PAGE_COUNT" }, { "name": "JWT_SECRET", "valueFrom": "PROD_JWT_SECRET" }, + { "name": "KEYCDN_API_KEY", "valueFrom": "PROD_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "PROD_MAX_NUM_OTP_ATTEMPTS" diff --git a/.aws/deploy/support-task-definition.staging.json b/.aws/deploy/support-task-definition.staging.json index dc77dbed4..c77efa8fc 100644 --- a/.aws/deploy/support-task-definition.staging.json +++ b/.aws/deploy/support-task-definition.staging.json @@ -107,6 +107,7 @@ "valueFrom": "STAGING_ISOMERPAGES_REPO_PAGE_COUNT" }, { "name": "JWT_SECRET", "valueFrom": "STAGING_JWT_SECRET" }, + { "name": "KEYCDN_API_KEY", "valueFrom": "STAGING_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "STAGING_MAX_NUM_OTP_ATTEMPTS" diff --git a/.env.test b/.env.test index 16075fe12..5d4a4ec13 100644 --- a/.env.test +++ b/.env.test @@ -85,3 +85,5 @@ export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" # GrowthBook export GROWTHBOOK_CLIENT_KEY="some random key" + +export KEYCDN_API_KEY="secret" \ No newline at end of file diff --git a/common/index.ts b/common/index.ts index 21f22bbc1..d0e7e976c 100644 --- a/common/index.ts +++ b/common/index.ts @@ -27,6 +27,7 @@ import { Reviewer, ReviewRequestView, } from "@database/models" +import MonitoringService from "@root/monitoring" import AuditLogsService from "@root/services/admin/AuditLogsService" import RepoManagementService from "@root/services/admin/RepoManagementService" import GitFileCommitService from "@root/services/db/GitFileCommitService" @@ -248,3 +249,7 @@ export const auditLogsService = new AuditLogsService({ sitesService, usersService, }) + +export const monitoringService = new MonitoringService({ + launchesService, +}) diff --git a/package-lock.json b/package-lock.json index d8dcd7f06..cce7c7ed4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@aws-sdk/lib-dynamodb": "^3.577.0", "@growthbook/growthbook": "^0.36.0", "@octokit/plugin-retry": "^6.0.0", - "@octokit/rest": "^18.12.0", + "@octokit/rest": "^20.1.1", "@opengovsg/formsg-sdk": "^0.11.0", "@opengovsg/sgid-client": "^2.0.0", "@slack/bolt": "^3.19.0", @@ -4519,7 +4519,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", - "peer": true, "engines": { "node": ">= 18" } @@ -4528,7 +4527,6 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -4545,14 +4543,12 @@ "node_modules/@octokit/core/node_modules/@octokit/openapi-types": { "version": "19.1.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", - "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==", - "peer": true + "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==" }, "node_modules/@octokit/core/node_modules/@octokit/types": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.4.0.tgz", "integrity": "sha512-FLWs/AvZllw/AGVs+nJ+ELCDZZJk+kY0zMen118xhL2zD0s1etIUHm1odgjP7epxYU1ln7SZxEUWYop5bhsdgQ==", - "peer": true, "dependencies": { "@octokit/openapi-types": "^19.1.0" } @@ -4561,7 +4557,6 @@ "version": "9.0.4", "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.4.tgz", "integrity": "sha512-DWPLtr1Kz3tv8L0UvXTDP1fNwM0S+z6EJpRcvH66orY6Eld4XBMCSYsaWp4xIm61jTWxK68BrR7ibO+vSDnZqw==", - "peer": true, "dependencies": { "@octokit/types": "^12.0.0", "universal-user-agent": "^6.0.0" @@ -4573,14 +4568,12 @@ "node_modules/@octokit/endpoint/node_modules/@octokit/openapi-types": { "version": "19.1.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", - "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==", - "peer": true + "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==" }, "node_modules/@octokit/endpoint/node_modules/@octokit/types": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.4.0.tgz", "integrity": "sha512-FLWs/AvZllw/AGVs+nJ+ELCDZZJk+kY0zMen118xhL2zD0s1etIUHm1odgjP7epxYU1ln7SZxEUWYop5bhsdgQ==", - "peer": true, "dependencies": { "@octokit/openapi-types": "^19.1.0" } @@ -4589,7 +4582,6 @@ "version": "7.0.2", "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz", "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==", - "peer": true, "dependencies": { "@octokit/request": "^8.0.1", "@octokit/types": "^12.0.0", @@ -4602,14 +4594,12 @@ "node_modules/@octokit/graphql/node_modules/@octokit/openapi-types": { "version": "19.1.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", - "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==", - "peer": true + "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==" }, "node_modules/@octokit/graphql/node_modules/@octokit/types": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.4.0.tgz", "integrity": "sha512-FLWs/AvZllw/AGVs+nJ+ELCDZZJk+kY0zMen118xhL2zD0s1etIUHm1odgjP7epxYU1ln7SZxEUWYop5bhsdgQ==", - "peer": true, "dependencies": { "@octokit/openapi-types": "^19.1.0" } @@ -4617,37 +4607,72 @@ "node_modules/@octokit/openapi-types": { "version": "12.11.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", - "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==", + "dev": true }, "node_modules/@octokit/plugin-paginate-rest": { - "version": "2.21.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", - "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-11.3.1.tgz", + "integrity": "sha512-ryqobs26cLtM1kQxqeZui4v8FeznirUsksiA+RYemMPJ7Micju0WSkv50dBksTuZks9O5cg4wp+t8fZ/cLY56g==", "dependencies": { - "@octokit/types": "^6.40.0" + "@octokit/types": "^13.5.0" + }, + "engines": { + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=2" + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "13.5.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.5.0.tgz", + "integrity": "sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" } }, "node_modules/@octokit/plugin-request-log": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz", - "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-4.0.1.tgz", + "integrity": "sha512-GihNqNpGHorUrO7Qa9JbAl0dbLnqJVrV8OXe2Zm5/Y4wFkZQDfTreBzVmiRfJVfE4mClXdihHnbpyyO9FSX4HA==", + "engines": { + "node": ">= 18" + }, "peerDependencies": { - "@octokit/core": ">=3" + "@octokit/core": "5" } }, "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "5.16.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", - "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-13.2.2.tgz", + "integrity": "sha512-EI7kXWidkt3Xlok5uN43suK99VWqc8OaIMktY9d9+RNKl69juoTyxmLoWPIZgJYzi41qj/9zU7G/ljnNOJ5AFA==", "dependencies": { - "@octokit/types": "^6.39.0", - "deprecation": "^2.3.1" + "@octokit/types": "^13.5.0" + }, + "engines": { + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=3" + "@octokit/core": "^5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.2.0.tgz", + "integrity": "sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "13.5.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.5.0.tgz", + "integrity": "sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==", + "dependencies": { + "@octokit/openapi-types": "^22.2.0" } }, "node_modules/@octokit/plugin-retry": { @@ -4683,7 +4708,6 @@ "version": "8.1.6", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.6.tgz", "integrity": "sha512-YhPaGml3ncZC1NfXpP3WZ7iliL1ap6tLkAp6MvbK2fTTPytzVUyUesBBogcdMm86uRYO5rHaM1xIWxigWZ17MQ==", - "peer": true, "dependencies": { "@octokit/endpoint": "^9.0.0", "@octokit/request-error": "^5.0.0", @@ -4723,98 +4747,35 @@ "node_modules/@octokit/request/node_modules/@octokit/openapi-types": { "version": "19.1.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", - "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==", - "peer": true + "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==" }, "node_modules/@octokit/request/node_modules/@octokit/types": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.4.0.tgz", "integrity": "sha512-FLWs/AvZllw/AGVs+nJ+ELCDZZJk+kY0zMen118xhL2zD0s1etIUHm1odgjP7epxYU1ln7SZxEUWYop5bhsdgQ==", - "peer": true, "dependencies": { "@octokit/openapi-types": "^19.1.0" } }, "node_modules/@octokit/rest": { - "version": "18.12.0", - "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-18.12.0.tgz", - "integrity": "sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q==", - "dependencies": { - "@octokit/core": "^3.5.1", - "@octokit/plugin-paginate-rest": "^2.16.8", - "@octokit/plugin-request-log": "^1.0.4", - "@octokit/plugin-rest-endpoint-methods": "^5.12.0" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/auth-token": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", - "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", - "dependencies": { - "@octokit/types": "^6.0.3" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/core": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", - "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", - "dependencies": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.3", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "dependencies": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/graphql": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", - "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", - "dependencies": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/request": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", - "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", - "dependencies": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.7", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/rest/node_modules/@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "version": "20.1.1", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-20.1.1.tgz", + "integrity": "sha512-MB4AYDsM5jhIHro/dq4ix1iWTLGToIGk6cWF5L6vanFaMble5jTX/UBQyiv05HsWnwUtY8JrfHy2LWfKwihqMw==", "dependencies": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" + "@octokit/core": "^5.0.2", + "@octokit/plugin-paginate-rest": "11.3.1", + "@octokit/plugin-request-log": "^4.0.0", + "@octokit/plugin-rest-endpoint-methods": "13.2.2" + }, + "engines": { + "node": ">= 18" } }, "node_modules/@octokit/types": { "version": "6.41.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "dev": true, "dependencies": { "@octokit/openapi-types": "^12.11.0" } @@ -12508,14 +12469,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", diff --git a/package.json b/package.json index c655eae21..e90cc5838 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@aws-sdk/lib-dynamodb": "^3.577.0", "@growthbook/growthbook": "^0.36.0", "@octokit/plugin-retry": "^6.0.0", - "@octokit/rest": "^18.12.0", + "@octokit/rest": "^20.1.1", "@opengovsg/formsg-sdk": "^0.11.0", "@opengovsg/sgid-client": "^2.0.0", "@slack/bolt": "^3.19.0", diff --git a/src/config/config.ts b/src/config/config.ts index 5cf91ce63..4b4d60888 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -195,6 +195,7 @@ const config = convict({ }, }, }, + github: { orgName: { doc: "GitHub organization that owns all site repositories", @@ -472,6 +473,14 @@ const config = convict({ default: "", }, }, + keyCdn: { + apiKey: { + doc: "KeyCDN API key", + env: "KEYCDN_API_KEY", + format: "required-string", + default: "", + }, + }, }) // Perform validation diff --git a/src/errors/MonitoringError.ts b/src/errors/MonitoringError.ts new file mode 100644 index 000000000..431aa5479 --- /dev/null +++ b/src/errors/MonitoringError.ts @@ -0,0 +1,11 @@ +import { BaseIsomerError } from "./BaseError" + +export default class MonitoringError extends BaseIsomerError { + constructor(message: string) { + super({ + status: 500, + code: "MonitoringError", + message, + }) + } +} diff --git a/src/monitoring/index.ts b/src/monitoring/index.ts new file mode 100644 index 000000000..de6e43ec0 --- /dev/null +++ b/src/monitoring/index.ts @@ -0,0 +1,244 @@ +import dns from "dns/promises" + +import { retry } from "@octokit/plugin-retry" +import { Octokit } from "@octokit/rest" +import autoBind from "auto-bind" +import axios from "axios" +import _ from "lodash" +import { errAsync, okAsync, ResultAsync } from "neverthrow" + +import parentLogger from "@logger/logger" + +import config from "@root/config/config" +import MonitoringError from "@root/errors/MonitoringError" +import LaunchesService from "@root/services/identity/LaunchesService" +import promisifyPapaParse from "@root/utils/papa-parse" + +interface MonitoringServiceProps { + launchesService: LaunchesService +} + +const IsomerHostedDomainType = { + REDIRECTION: "redirection", + INDIRECTION: "indirection", + KEYCDN: "keycdn", + AMPLIFY: "amplify", +} as const + +interface IsomerHostedDomain { + domain: string + type: typeof IsomerHostedDomainType[keyof typeof IsomerHostedDomainType] +} + +type KeyCdnZoneAlias = { + name: string +} + +interface RedirectionDomain { + source: string + target: string +} + +interface ReportCard { + domain: string + type: typeof IsomerHostedDomainType[keyof typeof IsomerHostedDomainType] + aRecord: string[] + quadArecord: string[] + cNameRecord: string[] + caaRecord: string[] +} + +function isKeyCdnZoneAlias(object: unknown): object is KeyCdnZoneAlias { + return "name" in (object as KeyCdnZoneAlias) +} + +function isKeyCdnResponse(object: unknown): object is KeyCdnZoneAlias[] { + if (!object) return false + if (Array.isArray(object)) return object.every(isKeyCdnZoneAlias) + return false +} + +export default class MonitoringService { + private readonly launchesService: MonitoringServiceProps["launchesService"] + + private readonly monitoringServiceLogger = parentLogger.child({ + module: "monitoringService", + }) + + constructor({ launchesService }: MonitoringServiceProps) { + autoBind(this) + this.launchesService = launchesService + } + + getKeyCdnDomains() { + const keyCdnApiKey = config.get("keyCdn.apiKey") + + return ResultAsync.fromPromise( + axios.get(`https://api.keycdn.com/zonealiases.json`, { + headers: { + Authorization: `Basic ${btoa(`${keyCdnApiKey}:`)}`, + }, + }), + (error) => + new MonitoringError(`Failed to fetch zones from KeyCDN: ${error}`) + ) + .map((response) => response.data.data.zonealiases) + .andThen((data) => { + if (!isKeyCdnResponse(data)) { + return errAsync( + new MonitoringError("Failed to parse response from KeyCDN") + ) + } + + const domains = data + .map((zone) => zone.name) + .map((domain) => ({ + domain, + type: IsomerHostedDomainType.KEYCDN, + })) + return okAsync(domains) + }) + } + + getAmplifyDeployments() { + return this.launchesService.getAllDomains().map((domains) => + domains.map((domain) => ({ + domain, + type: IsomerHostedDomainType.AMPLIFY, + })) + ) + } + + /** + * While most of our redirections are in our DB, we do have ad-hoc redirections. + * @returns List of redirection domains that are listed in the isomer-redirection repository + */ + getRedirectionDomains() { + const SYSTEM_GITHUB_TOKEN = config.get("github.systemToken") + const OctokitRetry = Octokit.plugin(retry) + const octokitWithRetry: Octokit = new OctokitRetry({ + auth: SYSTEM_GITHUB_TOKEN, + request: { retries: 5 }, + }) + + return ResultAsync.fromPromise( + octokitWithRetry.request( + "GET /repos/opengovsg/isomer-redirection/contents/src/certbot-websites.csv" + ), + (error) => + new MonitoringError(`Failed to fetch redirection domains: ${error}`) + ) + .andThen((response) => { + const content = Buffer.from(response.data.content, "base64").toString( + "utf-8" + ) + + return ResultAsync.fromPromise( + promisifyPapaParse(content), + (error) => new MonitoringError(`Failed to parse csv: ${error}`) + ) + }) + .map((redirectionDomains) => + redirectionDomains + .map((domain) => domain.source) + .map((domain) => ({ + domain, + type: IsomerHostedDomainType.REDIRECTION, + })) + ) + } + + /** + * This is in charge of fetching all the domains that are are under Isomer, inclusive + * of any subdomains and redirects. + */ + getAllDomains() { + this.monitoringServiceLogger.info("Fetching all domains") + return ResultAsync.combine([ + this.getAmplifyDeployments().mapErr( + (error) => new MonitoringError(error.message) + ), + this.getRedirectionDomains(), + this.getKeyCdnDomains(), + ]).andThen(([amplifyDeployments, redirectionDomains, keyCdnDomains]) => { + this.monitoringServiceLogger.info("Fetched all domains") + return okAsync( + _.sortBy( + [...amplifyDeployments, ...redirectionDomains, ...keyCdnDomains], + (val) => (val.domain.startsWith("www.") ? val.domain.slice(4) : val) + ) + ) + }) + } + + // todo: once /siteup logic is merged into dev, we can add that as to alert isomer team + generateReportCard(domains: IsomerHostedDomain[]) { + const reportCard: ReportCard[] = [] + + const domainResolvers = domains.map(({ domain, type }) => { + const aRecord = ResultAsync.fromPromise( + dns.resolve(domain, "A"), + (e) => e + ).orElse(() => okAsync([])) + const quadArecord = ResultAsync.fromPromise( + dns.resolve(domain, "AAAA"), + (e) => e + ).orElse(() => okAsync([])) + + const cNameRecord = ResultAsync.fromPromise( + dns.resolve(domain, "CNAME"), + (e) => e + ).orElse(() => okAsync([])) + + const caaRecord = ResultAsync.fromPromise( + dns.resolve(domain, "CAA"), + (e) => e + ) + .orElse(() => okAsync([])) + .map((records) => records.map((record) => record.toString())) + + return ResultAsync.combineWithAllErrors([ + aRecord, + quadArecord, + cNameRecord, + caaRecord, + ]) + .andThen((resolvedDns) => + okAsync({ + domain, + type, + aRecord: resolvedDns[0], + quadArecord: resolvedDns[1], + cNameRecord: resolvedDns[2], + caaRecord: resolvedDns[3], + }) + ) + .map((value) => + reportCard.push({ + ...value, + }) + ) + .andThen(() => okAsync(reportCard)) + }) + + return ResultAsync.combineWithAllErrors(domainResolvers).map( + () => reportCard + ) + } + + driver() { + this.monitoringServiceLogger.info("Monitoring service started") + return this.getAllDomains() + .andThen(this.generateReportCard) + .andThen((reportCard) => { + this.monitoringServiceLogger.info({ + message: "Report card generated", + meta: { + reportCard, + date: new Date(), + }, + }) + return okAsync(reportCard) + }) + } +} diff --git a/src/server.ts b/src/server.ts index 204637a20..4c1614333 100644 --- a/src/server.ts +++ b/src/server.ts @@ -80,6 +80,7 @@ import { mailer } from "@services/utilServices/MailClient" import { apiLogger } from "./middleware/apiLogger" import { NotificationOnEditHandler } from "./middleware/notificationOnEditHandler" +import MonitoringService from "./monitoring" import getAuthenticatedSubrouter from "./routes/v2/authenticated" import { ReviewsRouter } from "./routes/v2/authenticated/review" import getAuthenticatedSitesSubrouter from "./routes/v2/authenticatedSites" diff --git a/src/services/identity/LaunchesService.ts b/src/services/identity/LaunchesService.ts index 1f6c30715..4e713d648 100644 --- a/src/services/identity/LaunchesService.ts +++ b/src/services/identity/LaunchesService.ts @@ -353,6 +353,12 @@ export class LaunchesService { new SiteLaunchError(`Failed to update site status for ${siteName}`) ) }) + + getAllDomains = () => + ResultAsync.fromPromise( + this.launchesRepository.findAll(), + () => new SiteLaunchError("Failed to fetch launches") + ).map((launch) => launch.map((l) => l.primaryDomainSource)) } export default LaunchesService diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 7175b5648..ac07c2213 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -25,7 +25,7 @@ import { doesDirectoryExist } from "@root/utils/fs-utils" const SYSTEM_GITHUB_TOKEN = config.get("github.systemToken") const octokit = new Octokit({ auth: SYSTEM_GITHUB_TOKEN }) -const OctokitRetry = Octokit.plugin(retry as any) +const OctokitRetry = Octokit.plugin(retry) const octokitWithRetry = new OctokitRetry({ auth: SYSTEM_GITHUB_TOKEN, request: { retries: 5 }, diff --git a/src/utils/papa-parse.ts b/src/utils/papa-parse.ts new file mode 100644 index 000000000..e40723e20 --- /dev/null +++ b/src/utils/papa-parse.ts @@ -0,0 +1,19 @@ +import Papa from "papaparse" + +export default function promisifyPapaParse(content: string) { + return new Promise((resolve, reject) => { + Papa.parse(content, { + header: true, + complete(results) { + // validate the csv + if (!results.data) { + reject(new Error("Failed to parse csv")) + } + resolve(results.data as T) + }, + error(error: unknown) { + reject(error) + }, + }) + }) +} diff --git a/support/index.ts b/support/index.ts index ee0dffcd9..3c8e454d5 100644 --- a/support/index.ts +++ b/support/index.ts @@ -2,10 +2,16 @@ import "module-alias/register" import express from "express" -import { infraService, sequelize } from "@common/index" +import { + infraService, + launchesService, + monitoringService, + sequelize, +} from "@common/index" import { useSharedMiddleware } from "@common/middleware" import { config } from "@root/config/config" import logger from "@root/logger/logger" +import MonitoringService from "@root/monitoring" import { ROUTE_VERSION } from "./constants" import { v2Router } from "./routes" @@ -18,6 +24,8 @@ const app = express() // poller site launch updates infraService.pollMessages() +monitoringService.driver() + const ROUTE_PREFIX_ISOBOT = `/${ROUTE_VERSION}/isobot` app.use(ROUTE_PREFIX_ISOBOT, isobotRouter) From b565d582ffdfcb7cca48d38094f19ec9ad2b5c38 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Thu, 27 Jun 2024 14:14:39 +0800 Subject: [PATCH 5/7] feat(monitoring): add scheduler functionality (#1383) ## Problem This is the second part of the monitoring feature that we want to build. This PR only cares about adding a scheduler + the related infra needed for this to function. this will make the monitor run once every 5 mins, for oncalls to pick any related alarms from this. Adding the alarms is done in the downstream PR . ## Solution Using bullmq to conveniently create a queue, a worker and a repeatable job over multiple instances. We do some level of exponential backoff retries since it is a nice to have and easy to implement. The original `/site-up` code has since been refactored to return an `err` or a `ok`, depending on whether the configuration is ideal. Unfortunately, this caused quite a number of edge cases to pop up. Due to the nature of this, a more loose check of whether the isomer logo is present is being used to determine if a site is up. Even with this loose check, we have a `workplacelearning.gov.sg` who have modified their site to not have the Isomer logo. Have used gb to code white list this weird site. Potentially, if tomorrow we have an alarm of a site going down, but this is expected to prolong, we can go to growthbook and change the config for this to be whitelisted. **Breaking Changes** - [ ] Yes - this PR contains breaking changes - Details ... - [X] No - this PR is backwards compatible with ALL of the following feature flags in this [doc](https://www.notion.so/opengov/Existing-feature-flags-518ad2cdc325420893a105e88c432be5) ## Tests Screenshot 2024-05-21 at 11 20 59 AM on deployment, assert that you see these logs. it is ok for there to be multiple instances of this log (it directly corresponds to the number of instances that we have) since bullmq is smart enough to only create one queue, and one repeatable job over multiple instances. ## Deploy Notes corresponding infra pr should be deployed to production and only then should the redis host value be populated into the 1pw for production. Additionally, post approval of this pr, add two alarms, one for `Error running monitoring service` and another for `Monitoring service has failed`. These are errors when the job has failed to be initalised, and when there is a new error. **New environment variables**: - `REDIS_HOST` : Redis host - [ ] added env var to 1PW + SSM script (`fetch_ssm_parameters.sh`) **New dependencies**: - `bullmq` : scheduler of choice --- .aws/deploy/support-task-definition.prod.json | 5 + .../support-task-definition.staging.json | 5 + .env.test | 4 +- common/index.ts | 5 - docker-compose.dev.yml | 9 + package-lock.json | 257 ++++++++++++++ package.json | 1 + src/config/config.ts | 18 +- src/constants/constants.ts | 1 + src/constants/featureFlags.ts | 1 + src/monitoring/MonitoringService.ts | 116 ++++++ .../{index.ts => MonitoringWorker.ts} | 156 ++++----- src/server.ts | 2 +- src/types/featureFlags.ts | 6 + src/utils/dns-utils.ts | 329 ++++++++++++++++++ src/utils/growthbook-utils.ts | 20 +- support/index.ts | 19 +- support/routes/v2/isobot/ops/botService.ts | 310 +---------------- 18 files changed, 859 insertions(+), 405 deletions(-) create mode 100644 src/monitoring/MonitoringService.ts rename src/monitoring/{index.ts => MonitoringWorker.ts} (61%) create mode 100644 src/utils/dns-utils.ts diff --git a/.aws/deploy/support-task-definition.prod.json b/.aws/deploy/support-task-definition.prod.json index 7db06bc19..f5b5b5863 100644 --- a/.aws/deploy/support-task-definition.prod.json +++ b/.aws/deploy/support-task-definition.prod.json @@ -128,6 +128,11 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN" + }, + { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/support-task-definition.staging.json b/.aws/deploy/support-task-definition.staging.json index c77efa8fc..e8e21b0ab 100644 --- a/.aws/deploy/support-task-definition.staging.json +++ b/.aws/deploy/support-task-definition.staging.json @@ -137,6 +137,11 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN" + }, + { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" diff --git a/.env.test b/.env.test index 5d4a4ec13..1228238d9 100644 --- a/.env.test +++ b/.env.test @@ -86,4 +86,6 @@ export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" # GrowthBook export GROWTHBOOK_CLIENT_KEY="some random key" -export KEYCDN_API_KEY="secret" \ No newline at end of file +export KEYCDN_API_KEY="secret" + +export REDIS_HOST="redis" diff --git a/common/index.ts b/common/index.ts index d0e7e976c..21f22bbc1 100644 --- a/common/index.ts +++ b/common/index.ts @@ -27,7 +27,6 @@ import { Reviewer, ReviewRequestView, } from "@database/models" -import MonitoringService from "@root/monitoring" import AuditLogsService from "@root/services/admin/AuditLogsService" import RepoManagementService from "@root/services/admin/RepoManagementService" import GitFileCommitService from "@root/services/db/GitFileCommitService" @@ -249,7 +248,3 @@ export const auditLogsService = new AuditLogsService({ sitesService, usersService, }) - -export const monitoringService = new MonitoringService({ - launchesService, -}) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 046c0eae1..31896b87f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,10 +1,18 @@ version: "3" services: + redis: + image: redis + container_name: isomercms-redis + ports: + - 6379:6379 support: build: dockerfile: ./support/Dockerfile ports: - "8082:8082" + depends_on: + - postgres + - redis env_file: - .env volumes: @@ -18,6 +26,7 @@ services: dockerfile: Dockerfile depends_on: - postgres + - redis ports: - "8081:8081" env_file: diff --git a/package-lock.json b/package-lock.json index cce7c7ed4..b17ae63e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", + "bullmq": "^5.7.15", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", @@ -3312,6 +3313,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -4457,6 +4463,78 @@ "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==" }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -7459,6 +7537,32 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/bullmq": { + "version": "5.7.15", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz", + "integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -7804,6 +7908,14 @@ "superagent": "3.7.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -9130,6 +9242,17 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -9590,6 +9713,14 @@ "dev": true, "optional": true }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -12191,6 +12322,50 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", @@ -15675,6 +15850,11 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.find": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", @@ -15685,6 +15865,11 @@ "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -15876,6 +16061,14 @@ "node": ">=12" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -16379,6 +16572,35 @@ "msgpack": "bin/msgpack" } }, + "node_modules/msgpackr": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz", + "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -16531,6 +16753,17 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -18123,6 +18356,25 @@ "node": ">= 12.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -19181,6 +19433,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", diff --git a/package.json b/package.json index e90cc5838..c47a856f0 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", + "bullmq": "^5.7.15", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", diff --git a/src/config/config.ts b/src/config/config.ts index 4b4d60888..ee6ace999 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -195,7 +195,14 @@ const config = convict({ }, }, }, - + bullmq: { + redisHostname: { + doc: "Redis host name for bullmq", + env: "REDIS_HOST", + format: String, + default: "isomerpages", + }, + }, github: { orgName: { doc: "GitHub organization that owns all site repositories", @@ -247,6 +254,13 @@ const config = convict({ format: "required-string", default: "", }, + redirectionRepoGithubToken: { + doc: "Github access to read opengovsg/isomer-redirection", + env: "REDIRECTION_REPO_GITHUB_TOKEN", + sensitive: true, + format: String, + default: "", + }, }, dataDog: { env: { @@ -477,7 +491,7 @@ const config = convict({ apiKey: { doc: "KeyCDN API key", env: "KEYCDN_API_KEY", - format: "required-string", + format: String, default: "", }, }, diff --git a/src/constants/constants.ts b/src/constants/constants.ts index c3fc8743a..c3307fc42 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -78,6 +78,7 @@ export const REDIRECTION_SERVER_IPS = [ ] export const ALLOWED_DNS_ERROR_CODES = ["ENOTFOUND", "ENODATA"] +export const BUILT_WITH_ISOMER_LOGO = `