From 8729a9af4a366f1f5176c2868b4b23906706f805 Mon Sep 17 00:00:00 2001 From: shirady <57721533+shirady@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:33:41 +0200 Subject: [PATCH 1/6] NC | NSFS | Disable Prometheus reporting 1. The function clusterMetrics might throw an error (with an error message Operation timed out.), in case it fails, there is no try-catch block, and will end with uncaughtException. To avoid that we wrap it with try-catch block, log an error message and return from the function (we can't proceed with undefined metrics variable). Also added a json response with the details of the error in case it happens. 2. I noticed that the call prom_reporting.start_server(metrics_port, true); in fork_utils.js did not have await and added it - had to change the function signature of start_workers to have async and update its JSDoc @returns part, and also separate the call and the condition in endpoint.js. 3. Add a comment in the endpoint main for developers regarding implementation (running on the main process, running with multiple forks). 4. Change the handing under the function gather_metrics so we will print the json error object that we returned from the server. 5. Change the printing of "_create_nsfs_report: nsfs_report" so we can see the object (we used to see: "core.server.analytic_services.prometheus_reporting:: _create_nsfs_report: nsfs_report [object Object]"). 6. Rename a function (we had a minor typo) from nsfs_io_state_handler to nsfs_io_stats_handler. 7. Change the order inside the function start_server in prometheus_reporting.js and set the if (req.url === '/metrics/nsfs_stats') { because it doesn't use metrics. Signed-off-by: shirady <57721533+shirady@users.noreply.github.com> (cherry picked from commit 09c4c456cd0069606cf373b623662af3ea3e8fb5) --- src/endpoint/endpoint.js | 12 ++++++++- src/manage_nsfs/diagnose.js | 12 +++++++-- .../analytic_services/prometheus_reporting.js | 27 ++++++++++++++----- src/util/fork_utils.js | 10 +++---- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/endpoint/endpoint.js b/src/endpoint/endpoint.js index 3025bc05f1..c4c817500e 100755 --- a/src/endpoint/endpoint.js +++ b/src/endpoint/endpoint.js @@ -118,7 +118,17 @@ async function main(options = {}) { // the primary just forks and returns, workers will continue to serve fork_count = options.forks ?? config.ENDPOINT_FORKS; const metrics_port = options.metrics_port || config.EP_METRICS_SERVER_PORT; - if (fork_utils.start_workers(metrics_port, fork_count)) return; + /** + * Please notice that we can run the main in 2 states: + * 1. Only the primary process runs the main (fork is 0 or undefined) - everything that + * is implemented here would be run by this process. + * 2. A primary process with multiple forks (IMPORTANT) - if there is implementation that + * in only relevant to the primary process it should be implemented in + * fork_utils.start_workers because the primary process returns after start_workers + * and the forks will continue executing the code lines in this function + * */ + const is_workers_started_from_primary = await fork_utils.start_workers(metrics_port, fork_count); + if (is_workers_started_from_primary) return; const endpoint_group_id = process.env.ENDPOINT_GROUP_ID || 'default-endpoint-group'; diff --git a/src/manage_nsfs/diagnose.js b/src/manage_nsfs/diagnose.js index 56d65266d2..2e87015b31 100644 --- a/src/manage_nsfs/diagnose.js +++ b/src/manage_nsfs/diagnose.js @@ -59,9 +59,17 @@ async function gather_metrics() { const buffer = await buffer_utils.read_stream_join(res); const body = buffer.toString('utf8'); metrics_output = JSON.parse(body); + if (!metrics_output) throw new Error('received empty metrics response', { cause: res.statusCode }); + write_stdout_response(ManageCLIResponse.MetricsStatus, metrics_output); + } else if (res.statusCode >= 500 && res.rawHeaders.includes('application/json')) { + const buffer = await buffer_utils.read_stream_join(res); + const body = buffer.toString('utf8'); + const error_output = JSON.parse(body); + if (!error_output) throw new Error('received empty metrics response', { cause: res.statusCode }); + throw_cli_error({ ...ManageCLIError.MetricsStatusFailed, ...error_output }); + } else { + throw new Error('received empty metrics response', { cause: res.statusCode }); } - if (!metrics_output) throw new Error('recieved empty metrics response', { cause: res.statusCode }); - write_stdout_response(ManageCLIResponse.MetricsStatus, metrics_output); } catch (err) { dbg.warn('could not receive metrics response', err); throw_cli_error({ ...ManageCLIError.MetricsStatusFailed, cause: err?.errors?.[0] || err }); diff --git a/src/server/analytic_services/prometheus_reporting.js b/src/server/analytic_services/prometheus_reporting.js index 164e4d502b..caee6bf86b 100644 --- a/src/server/analytic_services/prometheus_reporting.js +++ b/src/server/analytic_services/prometheus_reporting.js @@ -61,12 +61,7 @@ async function start_server( const server = http.createServer(async (req, res) => { // Serve all metrics on the root path for system that do have one or more fork running. if (fork_enabled) { - const metrics = await aggregatorRegistry.clusterMetrics(); - if (req.url === '' || req.url === '/') { - res.writeHead(200, { 'Content-Type': aggregatorRegistry.contentType }); - res.end(metrics); - return; - } + // we would like this part to be first as clusterMetrics might fail. if (req.url === '/metrics/nsfs_stats') { res.writeHead(200, { 'Content-Type': 'text/plain' }); const nsfs_report = { @@ -77,6 +72,24 @@ async function start_server( res.end(JSON.stringify(nsfs_report)); return; } + let metrics; + try { + metrics = await aggregatorRegistry.clusterMetrics(); + } catch (err) { + dbg.error('start_server: Could not get the metrics, got an error', err); + res.writeHead(504, { 'Content-Type': 'application/json' }); + const reply = JSON.stringify({ + error: 'Internal server error - timeout', + message: 'Looks like the server is taking a long time to respond (Could not get the metrics)', + }); + res.end(reply); + return; + } + if (req.url === '' || req.url === '/') { + res.writeHead(200, { 'Content-Type': aggregatorRegistry.contentType }); + res.end(metrics); + return; + } // Serve report's metrics on the report name path const report_name = req.url.substr(1); const single_metrics = export_single_metrics(metrics, report_name); @@ -165,7 +178,7 @@ async function metrics_nsfs_stats_handler() { op_stats_counters: op_stats_counters, fs_worker_stats_counters: fs_worker_stats_counters }; - dbg.log1(`_create_nsfs_report: nsfs_report ${nsfs_report}`); + dbg.log1('_create_nsfs_report: nsfs_report', nsfs_report); return JSON.stringify(nsfs_report); } diff --git a/src/util/fork_utils.js b/src/util/fork_utils.js index 5b6e6378e8..52c00d7b60 100644 --- a/src/util/fork_utils.js +++ b/src/util/fork_utils.js @@ -30,9 +30,9 @@ const fs_workers_stats = {}; * * @param {number?} count number of workers to start. * @param {number?} metrics_port prometheus metris port. - * @returns {boolean} true if workers were started. + * @returns {Promise} true if workers were started. */ -function start_workers(metrics_port, count = 0) { +async function start_workers(metrics_port, count = 0) { const exit_events = []; if (cluster.isPrimary && count > 0) { for (let i = 0; i < count; ++i) { @@ -68,12 +68,12 @@ function start_workers(metrics_port, count = 0) { }); for (const id in cluster.workers) { if (id) { - cluster.workers[id].on('message', nsfs_io_state_handler); + cluster.workers[id].on('message', nsfs_io_stats_handler); } } if (metrics_port > 0) { dbg.log0('Starting metrics server', metrics_port); - prom_reporting.start_server(metrics_port, true); + await prom_reporting.start_server(metrics_port, true); dbg.log0('Started metrics server successfully'); } return true; @@ -82,7 +82,7 @@ function start_workers(metrics_port, count = 0) { return false; } -function nsfs_io_state_handler(msg) { +function nsfs_io_stats_handler(msg) { if (msg.io_stats) { for (const [key, value] of Object.entries(msg.io_stats)) { io_stats[key] += value; From dd5fb6fabff739e6b955e1d695877ab646dc1a95 Mon Sep 17 00:00:00 2001 From: liranmauda Date: Wed, 27 Nov 2024 11:12:11 +0200 Subject: [PATCH 2/6] Bumping deps to avoid CVE (01/12/2024) - Bumping deps to avoid CVE (01/12/2024) Signed-off-by: liranmauda (cherry picked from commit d5507566dd33587d3e91c3fa782d86556269ff9b) --- package-lock.json | 1156 ++++++++++++++++++++++++++++++++------------- package.json | 19 +- 2 files changed, 842 insertions(+), 333 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a943932a9..6ae8bcbfcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,12 +11,12 @@ "license": "SEE LICENSE IN LICENSE", "dependencies": { "@aws-sdk/client-s3": "3.689.0", - "@aws-sdk/client-sts": "3.687.0", + "@aws-sdk/client-sts": "3.699.0", "@azure/identity": "4.5.0", "@azure/monitor-query": "1.3.1", "@azure/storage-blob": "12.25.0", "@google-cloud/storage": "7.14.0", - "@smithy/node-http-handler": "3.2.5", + "@smithy/node-http-handler": "3.3.1", "ajv": "8.17.1", "aws-sdk": "2.1692.0", "bcrypt": "5.1.1", @@ -46,8 +46,8 @@ "morgan": "1.10.0", "nan": "2.22.0", "ncp": "2.0.0", - "node-addon-api": "8.2.2", - "node-rdkafka": "3.1.1", + "node-addon-api": "8.3.0", + "node-rdkafka": "3.2.0", "performance-now": "2.1.0", "pg": "8.13.1", "ping": "0.4.4", @@ -56,14 +56,14 @@ "rimraf": "6.0.1", "seedrandom": "3.0.5", "setimmediate": "1.0.5", - "typescript": "5.6.3", + "typescript": "5.7.2", "utf-8-validate": "6.0.5", "uuid": "10.0.0", "ws": "8.18.0", "xml2js": "0.6.2", - "yaml": "2.6.0", + "yaml": "2.6.1", "yauzl": "3.2.0", - "yazl": "2.5.1" + "yazl": "3.3.1" }, "bin": { "noobaa-core": "src/cmd/index.js" @@ -74,9 +74,8 @@ "@types/jest": "29.5.14", "@types/lodash": "4.17.13", "@types/mongodb": "4.0.7", - "@types/node": "22.4.1", + "@types/node": "22.10.0", "@types/pg": "8.11.10", - "@types/request": "2.48.12", "eslint": "8.57.1", "eslint-plugin-header": "3.1.1", "eslint-plugin-jest": "28.9.0", @@ -86,7 +85,7 @@ "pkg": "5.8.1", "pkg-fetch": "3.5.2", "sinon": "19.0.2", - "wtfnode": "0.9.3" + "wtfnode": "0.9.4" } }, "node_modules/@ampproject/remapping": { @@ -374,15 +373,17 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso": { + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sts": { "version": "3.687.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.687.0.tgz", - "integrity": "sha512-dfj0y9fQyX4kFill/ZG0BqBTLQILKlL7+O5M4F9xlsh2WNuV2St6WtcOg14Y1j5UODPJiJs//pO+mD1lihT5Kw==", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.687.0.tgz", + "integrity": "sha512-SQjDH8O4XCTtouuCVYggB0cCCrIaTzUZIkgJUpOsIEJBLlTbNOb/BZqUShAQw2o9vxr2rCeOGjAQOYPysW/Pmg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.687.0", "@aws-sdk/core": "3.686.0", + "@aws-sdk/credential-provider-node": "3.687.0", "@aws-sdk/middleware-host-header": "3.686.0", "@aws-sdk/middleware-logger": "3.686.0", "@aws-sdk/middleware-recursion-detection": "3.686.0", @@ -423,16 +424,15 @@ "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso-oidc": { + "node_modules/@aws-sdk/client-sso": { "version": "3.687.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.687.0.tgz", - "integrity": "sha512-Rdd8kLeTeh+L5ZuG4WQnWgYgdv7NorytKdZsGjiag1D8Wv3PcJvPqqWdgnI0Og717BSXVoaTYaN34FyqFYSx6Q==", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.687.0.tgz", + "integrity": "sha512-dfj0y9fQyX4kFill/ZG0BqBTLQILKlL7+O5M4F9xlsh2WNuV2St6WtcOg14Y1j5UODPJiJs//pO+mD1lihT5Kw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.686.0", - "@aws-sdk/credential-provider-node": "3.687.0", "@aws-sdk/middleware-host-header": "3.686.0", "@aws-sdk/middleware-logger": "3.686.0", "@aws-sdk/middleware-recursion-detection": "3.686.0", @@ -471,20 +471,16 @@ }, "engines": { "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.687.0" } }, - "node_modules/@aws-sdk/client-sts": { + "node_modules/@aws-sdk/client-sso-oidc": { "version": "3.687.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.687.0.tgz", - "integrity": "sha512-SQjDH8O4XCTtouuCVYggB0cCCrIaTzUZIkgJUpOsIEJBLlTbNOb/BZqUShAQw2o9vxr2rCeOGjAQOYPysW/Pmg==", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.687.0.tgz", + "integrity": "sha512-Rdd8kLeTeh+L5ZuG4WQnWgYgdv7NorytKdZsGjiag1D8Wv3PcJvPqqWdgnI0Og717BSXVoaTYaN34FyqFYSx6Q==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/client-sso-oidc": "3.687.0", "@aws-sdk/core": "3.686.0", "@aws-sdk/credential-provider-node": "3.687.0", "@aws-sdk/middleware-host-header": "3.686.0", @@ -523,10 +519,491 @@ "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.687.0" + } + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.699.0.tgz", + "integrity": "sha512-++lsn4x2YXsZPIzFVwv3fSUVM55ZT0WRFmPeNilYIhZClxHLmVAWKH4I55cY9ry60/aTKYjzOXkWwyBKGsGvQg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/client-sso": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.696.0.tgz", + "integrity": "sha512-q5TTkd08JS0DOkHfUL853tuArf7NrPeqoS5UOvqJho8ibV9Ak/a/HO4kNvy9Nj3cib/toHYHsQIEtecUPSUUrQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.699.0.tgz", + "integrity": "sha512-u8a1GorY5D1l+4FQAf4XBUC1T10/t7neuwT21r0ymrtMFSK2a9QqVHKMoLkvavAwyhJnARSBM9/UQC797PFOFw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.696.0.tgz", + "integrity": "sha512-T9iMFnJL7YTlESLpVFT3fg1Lkb1lD+oiaIC8KMpepb01gDUBIpj9+Y+pA/cgRWW0yRxmkDXNazAE2qQTVFGJzA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.696.0.tgz", + "integrity": "sha512-GV6EbvPi2eq1+WgY/o2RFA3P7HGmnkIzCNmhwtALFlqMroLYWKE7PSeHw66Uh1dFQeVESn0/+hiUNhu1mB0emA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.699.0.tgz", + "integrity": "sha512-dXmCqjJnKmG37Q+nLjPVu22mNkrGHY8hYoOt3Jo9R2zr5MYV7s/NHsCHr+7E+BZ+tfZYLRPeB1wkpTeHiEcdRw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.699.0.tgz", + "integrity": "sha512-MmEmNDo1bBtTgRmdNfdQksXu4uXe66s0p1hi1YPrn1h59Q605eq/xiWbGL6/3KdkViH6eGUuABeV2ODld86ylg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-ini": "3.699.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.696.0.tgz", + "integrity": "sha512-mL1RcFDe9sfmyU5K1nuFkO8UiJXXxLX4JO1gVaDIOvPqwStpUAwi3A1BoeZhWZZNQsiKI810RnYGo0E0WB/hUA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.699.0.tgz", + "integrity": "sha512-Ekp2cZG4pl9D8+uKWm4qO1xcm8/MeiI8f+dnlZm8aQzizeC+aXYy9GyoclSf6daK8KfRPiRfM7ZHBBL5dAfdMA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-sso": "3.696.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/token-providers": "3.699.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.696.0.tgz", + "integrity": "sha512-XJ/CVlWChM0VCoc259vWguFUjJDn/QwDqHwbx+K9cg3v6yrqXfK5ai+p/6lx0nQpnk4JzPVeYYxWRpaTsGC9rg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.696.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.696.0.tgz", + "integrity": "sha512-zELJp9Ta2zkX7ELggMN9qMCgekqZhFC5V2rOr4hJDEb/Tte7gpfKSObAnw/3AYiVqt36sjHKfdkoTsuwGdEoDg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/middleware-logger": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.696.0.tgz", + "integrity": "sha512-KhkHt+8AjCxcR/5Zp3++YPJPpFQzxpr+jmONiT/Jw2yqnSngZ0Yspm5wGoRx2hS1HJbyZNuaOWEGuJoxLeBKfA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.696.0.tgz", + "integrity": "sha512-si/maV3Z0hH7qa99f9ru2xpS5HlfSVcasRlNUXKSDm611i7jFMWwGNLUOXFAOLhXotPX5G3Z6BLwL34oDeBMug==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.696.0.tgz", + "integrity": "sha512-Lvyj8CTyxrHI6GHd2YVZKIRI5Fmnugt3cpJo0VrKKEgK5zMySwEZ1n4dqPK6czYRWKd5+WnYHYAuU+Wdk6Jsjw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.696.0.tgz", + "integrity": "sha512-7EuH142lBXjI8yH6dVS/CZeiK/WZsmb/8zP6bQbVYpMrppSTgB3MzZZdxVZGzL5r8zPQOU10wLC4kIMy0qdBVQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/token-providers": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.699.0.tgz", + "integrity": "sha512-kuiEW9DWs7fNos/SM+y58HCPhcIzm1nEZLhe2/7/6+TvAYLuEWURYsbK48gzsxXlaJ2k/jGY3nIsA7RptbMOwA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, "engines": { "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/util-endpoints": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.696.0.tgz", + "integrity": "sha512-T5s0IlBVX+gkb9g/I6CLt4yAZVzMSiGnbUqWihWsHvQR1WOoIcndQy/Oz/IJXT9T2ipoy7a80gzV6a5mglrioA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "@smithy/util-endpoints": "^2.1.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.696.0.tgz", + "integrity": "sha512-Z5rVNDdmPOe6ELoM5AhF/ja5tSjbe6ctSctDPb0JdDf4dT0v2MfwhJKzXju2RzX8Es/77Glh7MlaXLE0kCB9+Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.696.0.tgz", + "integrity": "sha512-KhKqcfyXIB0SCCt+qsu4eJjsfiOrNzK5dCV7RAW2YIpp+msxGUUX0NdRE9rkzjiv+3EMktgJm3eEIS+yxtlVdQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@aws-sdk/core": { "version": "3.686.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.686.0.tgz", @@ -978,9 +1455,9 @@ } }, "node_modules/@aws-sdk/util-locate-window": { - "version": "3.679.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.679.0.tgz", - "integrity": "sha512-zKTd48/ZWrCplkXpYDABI74rQlbR0DNHs8nH95htfSLj9/mWRSwaGptoxwcihaq/77vi/fl2X3y0a1Bo8bt7RA==", + "version": "3.693.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.693.0.tgz", + "integrity": "sha512-ttrag6haJLWABhLqtg1Uf+4LgHWIMOVSYL+VYZmAp2v4PUGOwWmWQH0Zk8RM7YuQcLfH/EoR72/Yxz6A4FKcuw==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -1124,15 +1601,15 @@ } }, "node_modules/@azure/core-rest-pipeline": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.17.0.tgz", - "integrity": "sha512-62Vv8nC+uPId3j86XJ0WI+sBf0jlqTqPUFCBNrGtlaUeQUIXWV/D8GE5A1d+Qx8H7OQojn2WguC8kChD6v0shA==", + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.1.tgz", + "integrity": "sha512-/wS73UEDrxroUEVywEm7J0p2c+IIiVxyfigCGfsKvCxxCET4V/Hef2aURqltrXMRjNmdmt5IuOgIpl8f6xdO5A==", "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.0.0", "@azure/core-auth": "^1.8.0", "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.9.0", + "@azure/core-util": "^1.11.0", "@azure/logger": "^1.0.0", "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.0", @@ -1258,9 +1735,9 @@ } }, "node_modules/@azure/msal-node": { - "version": "2.16.0", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-2.16.0.tgz", - "integrity": "sha512-oww0oJTOOvPKTVXqVyxfcFVjExQKYEkKR5KM0cTG3jnzt6u/MRMx8XaK49L/bxV35r9sCHQFjNlEShad9qGSYA==", + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-2.16.2.tgz", + "integrity": "sha512-An7l1hEr0w1HMMh1LU+rtDtqL7/jw74ORlc9Wnh06v7TU/xpG39/Zdr1ZJu3QpjUfKJ+E0/OXMW8DRSWTlh7qQ==", "license": "MIT", "dependencies": { "@azure/msal-common": "14.16.0", @@ -2912,12 +3389,12 @@ "license": "(Unlicense OR Apache-2.0)" }, "node_modules/@smithy/abort-controller": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.6.tgz", - "integrity": "sha512-0XuhuHQlEqbNQZp7QxxrFTdVWdwxch4vjxYgfInF91hZFkPxf9QDrdQka0KfxFMPqLNzSw0b95uGTrLliQUavQ==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -2944,15 +3421,15 @@ } }, "node_modules/@smithy/config-resolver": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.10.tgz", - "integrity": "sha512-Uh0Sz9gdUuz538nvkPiyv1DZRX9+D15EKDtnQP5rYVAzM/dnYk3P8cg73jcxyOitPgT3mE3OVj7ky7sibzHWkw==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.12.tgz", + "integrity": "sha512-YAJP9UJFZRZ8N+UruTeq78zkdjUHmzsY62J4qKWZ4SXB4QXJ/+680EfXXgkYA2xj77ooMqtUY9m406zGNqwivQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^3.1.9", - "@smithy/types": "^3.6.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.8", + "@smithy/util-middleware": "^3.0.10", "tslib": "^2.6.2" }, "engines": { @@ -2960,17 +3437,17 @@ } }, "node_modules/@smithy/core": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.5.1.tgz", - "integrity": "sha512-DujtuDA7BGEKExJ05W5OdxCoyekcKT3Rhg1ZGeiUWaz2BJIWXjZmsG/DIP4W48GHno7AQwRsaCb8NcBgH3QZpg==", + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.5.4.tgz", + "integrity": "sha512-iFh2Ymn2sCziBRLPuOOxRPkuCx/2gBdXtBGuCUFLUe6bWYjKnhHyIPqGeNkLZ5Aco/5GjebRTBFiWID3sDbrKw==", "license": "Apache-2.0", "dependencies": { - "@smithy/middleware-serde": "^3.0.8", - "@smithy/protocol-http": "^4.1.5", - "@smithy/types": "^3.6.0", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-middleware": "^3.0.8", - "@smithy/util-stream": "^3.2.1", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-stream": "^3.3.1", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" }, @@ -2979,15 +3456,15 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.5.tgz", - "integrity": "sha512-4FTQGAsuwqTzVMmiRVTn0RR9GrbRfkP0wfu/tXWVHd2LgNpTY0uglQpIScXK4NaEyXbB3JmZt8gfVqO50lP8wg==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.7.tgz", + "integrity": "sha512-cEfbau+rrWF8ylkmmVAObOmjbTIzKyUC5TkBL58SbLywD0RCBC4JAUKbmtSm2w5KUJNRPGgpGFMvE2FKnuNlWQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^3.1.9", - "@smithy/property-provider": "^3.1.8", - "@smithy/types": "^3.6.0", - "@smithy/url-parser": "^3.0.8", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", "tslib": "^2.6.2" }, "engines": { @@ -2995,25 +3472,25 @@ } }, "node_modules/@smithy/eventstream-codec": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.7.tgz", - "integrity": "sha512-kVSXScIiRN7q+s1x7BrQtZ1Aa9hvvP9FeCqCdBxv37GimIHgBCOnZ5Ip80HLt0DhnAKpiobFdGqTFgbaJNrazA==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.9.tgz", + "integrity": "sha512-F574nX0hhlNOjBnP+noLtsPFqXnWh2L0+nZKCwcu7P7J8k+k+rdIDs+RMnrMwrzhUE4mwMgyN0cYnEn0G8yrnQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "@smithy/util-hex-encoding": "^3.0.0", "tslib": "^2.6.2" } }, "node_modules/@smithy/eventstream-serde-browser": { - "version": "3.0.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.11.tgz", - "integrity": "sha512-Pd1Wnq3CQ/v2SxRifDUihvpXzirJYbbtXfEnnLV/z0OGCTx/btVX74P86IgrZkjOydOASBGXdPpupYQI+iO/6A==", + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.13.tgz", + "integrity": "sha512-Nee9m+97o9Qj6/XeLz2g2vANS2SZgAxV4rDBMKGHvFJHU/xz88x2RwCkwsvEwYjSX4BV1NG1JXmxEaDUzZTAtw==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.10", - "@smithy/types": "^3.6.0", + "@smithy/eventstream-serde-universal": "^3.0.12", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3021,12 +3498,12 @@ } }, "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.8.tgz", - "integrity": "sha512-zkFIG2i1BLbfoGQnf1qEeMqX0h5qAznzaZmMVNnvPZz9J5AWBPkOMckZWPedGUPcVITacwIdQXoPcdIQq5FRcg==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.10.tgz", + "integrity": "sha512-K1M0x7P7qbBUKB0UWIL5KOcyi6zqV5mPJoL0/o01HPJr0CSq3A9FYuJC6e11EX6hR8QTIR++DBiGrYveOu6trw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3034,13 +3511,13 @@ } }, "node_modules/@smithy/eventstream-serde-node": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.10.tgz", - "integrity": "sha512-hjpU1tIsJ9qpcoZq9zGHBJPBOeBGYt+n8vfhDwnITPhEre6APrvqq/y3XMDEGUT2cWQ4ramNqBPRbx3qn55rhw==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.12.tgz", + "integrity": "sha512-kiZymxXvZ4tnuYsPSMUHe+MMfc4FTeFWJIc0Q5wygJoUQM4rVHNghvd48y7ppuulNMbuYt95ah71pYc2+o4JOA==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.10", - "@smithy/types": "^3.6.0", + "@smithy/eventstream-serde-universal": "^3.0.12", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3048,13 +3525,13 @@ } }, "node_modules/@smithy/eventstream-serde-universal": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.10.tgz", - "integrity": "sha512-ewG1GHbbqsFZ4asaq40KmxCmXO+AFSM1b+DcO2C03dyJj/ZH71CiTg853FSE/3SHK9q3jiYQIFjlGSwfxQ9kww==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.12.tgz", + "integrity": "sha512-1i8ifhLJrOZ+pEifTlF0EfZzMLUGQggYQ6WmZ4d5g77zEKf7oZ0kvh1yKWHPjofvOwqrkwRDVuxuYC8wVd662A==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-codec": "^3.1.7", - "@smithy/types": "^3.6.0", + "@smithy/eventstream-codec": "^3.1.9", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3062,37 +3539,37 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.0.0.tgz", - "integrity": "sha512-MLb1f5tbBO2X6K4lMEKJvxeLooyg7guq48C2zKr4qM7F2Gpkz4dc+hdSgu77pCJ76jVqFBjZczHYAs6dp15N+g==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^4.1.5", - "@smithy/querystring-builder": "^3.0.8", - "@smithy/types": "^3.6.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", "@smithy/util-base64": "^3.0.0", "tslib": "^2.6.2" } }, "node_modules/@smithy/hash-blob-browser": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.1.7.tgz", - "integrity": "sha512-4yNlxVNJifPM5ThaA5HKnHkn7JhctFUHvcaz6YXxHlYOSIrzI6VKQPTN8Gs1iN5nqq9iFcwIR9THqchUCouIfg==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.1.9.tgz", + "integrity": "sha512-wOu78omaUuW5DE+PVWXiRKWRZLecARyP3xcq5SmkXUw9+utgN8HnSnBfrjL2B/4ZxgqPjaAJQkC/+JHf1ITVaQ==", "license": "Apache-2.0", "dependencies": { "@smithy/chunked-blob-reader": "^4.0.0", "@smithy/chunked-blob-reader-native": "^3.0.1", - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" } }, "node_modules/@smithy/hash-node": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.8.tgz", - "integrity": "sha512-tlNQYbfpWXHimHqrvgo14DrMAgUBua/cNoz9fMYcDmYej7MAmUcjav/QKQbFc3NrcPxeJ7QClER4tWZmfwoPng==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.10.tgz", + "integrity": "sha512-3zWGWCHI+FlJ5WJwx73Mw2llYR8aflVyZN5JhoqLxbdPZi6UyKSdCeXAWJw9ja22m6S6Tzz1KZ+kAaSwvydi0g==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "@smithy/util-buffer-from": "^3.0.0", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" @@ -3102,12 +3579,12 @@ } }, "node_modules/@smithy/hash-stream-node": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.1.7.tgz", - "integrity": "sha512-xMAsvJ3hLG63lsBVi1Hl6BBSfhd8/Qnp8fC06kjOpJvyyCEXdwHITa5Kvdsk6gaAXLhbZMhQMIGvgUbfnJDP6Q==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.1.9.tgz", + "integrity": "sha512-3XfHBjSP3oDWxLmlxnt+F+FqXpL3WlXs+XXaB6bV9Wo8BBu87fK1dSEsyH7Z4ZHRmwZ4g9lFMdf08m9hoX1iRA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" }, @@ -3116,12 +3593,12 @@ } }, "node_modules/@smithy/invalid-dependency": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.8.tgz", - "integrity": "sha512-7Qynk6NWtTQhnGTTZwks++nJhQ1O54Mzi7fz4PqZOiYXb4Z1Flpb2yRvdALoggTS8xjtohWUM+RygOtB30YL3Q==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.10.tgz", + "integrity": "sha512-Lp2L65vFi+cj0vFMu2obpPW69DU+6O5g3086lmI4XcnRCG8PxvpWC7XyaVwJCxsZFzueHjXnrOH/E0pl0zikfA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" } }, @@ -3138,24 +3615,24 @@ } }, "node_modules/@smithy/md5-js": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.8.tgz", - "integrity": "sha512-LwApfTK0OJ/tCyNUXqnWCKoE2b4rDSr4BJlDAVCkiWYeHESr+y+d5zlAanuLW6fnitVJRD/7d9/kN/ZM9Su4mA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.10.tgz", + "integrity": "sha512-m3bv6dApflt3fS2Y1PyWPUtRP7iuBlvikEOGwu0HsCZ0vE7zcIX+dBoh3e+31/rddagw8nj92j0kJg2TfV+SJA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" } }, "node_modules/@smithy/middleware-content-length": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.10.tgz", - "integrity": "sha512-T4dIdCs1d/+/qMpwhJ1DzOhxCZjZHbHazEPJWdB4GDi2HjIZllVzeBEcdJUN0fomV8DURsgOyrbEUzg3vzTaOg==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.12.tgz", + "integrity": "sha512-1mDEXqzM20yywaMDuf5o9ue8OkJ373lSPbaSjyEvkWdqELhFMyNNgKGWL/rCSf4KME8B+HlHKuR8u9kRj8HzEQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^4.1.5", - "@smithy/types": "^3.6.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3163,18 +3640,18 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.1.tgz", - "integrity": "sha512-wWO3xYmFm6WRW8VsEJ5oU6h7aosFXfszlz3Dj176pTij6o21oZnzkCLzShfmRaaCHDkBXWBdO0c4sQAvLFP6zA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^2.5.1", - "@smithy/middleware-serde": "^3.0.8", - "@smithy/node-config-provider": "^3.1.9", - "@smithy/shared-ini-file-loader": "^3.1.9", - "@smithy/types": "^3.6.0", - "@smithy/url-parser": "^3.0.8", - "@smithy/util-middleware": "^3.0.8", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.4.tgz", + "integrity": "sha512-TybiW2LA3kYVd3e+lWhINVu1o26KJbBwOpADnf0L4x/35vLVica77XVR5hvV9+kWeTGeSJ3IHTcYxbRxlbwhsg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-middleware": "^3.0.10", "tslib": "^2.6.2" }, "engines": { @@ -3182,18 +3659,18 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "3.0.25", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.25.tgz", - "integrity": "sha512-m1F70cPaMBML4HiTgCw5I+jFNtjgz5z5UdGnUbG37vw6kh4UvizFYjqJGHvicfgKMkDL6mXwyPp5mhZg02g5sg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^3.1.9", - "@smithy/protocol-http": "^4.1.5", - "@smithy/service-error-classification": "^3.0.8", - "@smithy/smithy-client": "^3.4.2", - "@smithy/types": "^3.6.0", - "@smithy/util-middleware": "^3.0.8", - "@smithy/util-retry": "^3.0.8", + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.28.tgz", + "integrity": "sha512-vK2eDfvIXG1U64FEUhYxoZ1JSj4XFbYWkK36iz02i3pFwWiDz1Q7jKhGTBCwx/7KqJNk4VS7d7cDLXFOvP7M+g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/service-error-classification": "^3.0.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", "tslib": "^2.6.2", "uuid": "^9.0.1" }, @@ -3215,12 +3692,12 @@ } }, "node_modules/@smithy/middleware-serde": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.8.tgz", - "integrity": "sha512-Xg2jK9Wc/1g/MBMP/EUn2DLspN8LNt+GMe7cgF+Ty3vl+Zvu+VeZU5nmhveU+H8pxyTsjrAkci8NqY6OuvZnjA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3228,12 +3705,12 @@ } }, "node_modules/@smithy/middleware-stack": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.8.tgz", - "integrity": "sha512-d7ZuwvYgp1+3682Nx0MD3D/HtkmZd49N3JUndYWQXfRZrYEnCWYc8BHcNmVsPAp9gKvlurdg/mubE6b/rPS9MA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.10.tgz", + "integrity": "sha512-grCHyoiARDBBGPyw2BeicpjgpsDFWZZxptbVKb3CRd/ZA15F/T6rZjCCuBUjJwdck1nwUuIxYtsS4H9DDpbP5w==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3241,14 +3718,14 @@ } }, "node_modules/@smithy/node-config-provider": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.9.tgz", - "integrity": "sha512-qRHoah49QJ71eemjuS/WhUXB+mpNtwHRWQr77J/m40ewBVVwvo52kYAmb7iuaECgGTTcYxHS4Wmewfwy++ueew==", + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^3.1.8", - "@smithy/shared-ini-file-loader": "^3.1.9", - "@smithy/types": "^3.6.0", + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3256,15 +3733,15 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.2.5.tgz", - "integrity": "sha512-PkOwPNeKdvX/jCpn0A8n9/TyoxjGZB8WVoJmm9YzsnAgggTj4CrjpRHlTQw7dlLZ320n1mY1y+nTRUDViKi/3w==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^3.1.6", - "@smithy/protocol-http": "^4.1.5", - "@smithy/querystring-builder": "^3.0.8", - "@smithy/types": "^3.6.0", + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3272,12 +3749,12 @@ } }, "node_modules/@smithy/property-provider": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.8.tgz", - "integrity": "sha512-ukNUyo6rHmusG64lmkjFeXemwYuKge1BJ8CtpVKmrxQxc6rhUX0vebcptFA9MmrGsnLhwnnqeH83VTU9hwOpjA==", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3285,12 +3762,12 @@ } }, "node_modules/@smithy/protocol-http": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.5.tgz", - "integrity": "sha512-hsjtwpIemmCkm3ZV5fd/T0bPIugW1gJXwZ/hpuVubt2hEUApIoUTrf6qIdh9MAWlw0vjMrA1ztJLAwtNaZogvg==", + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3298,12 +3775,12 @@ } }, "node_modules/@smithy/querystring-builder": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.8.tgz", - "integrity": "sha512-btYxGVqFUARbUrN6VhL9c3dnSviIwBYD9Rz1jHuN1hgh28Fpv2xjU1HeCeDJX68xctz7r4l1PBnFhGg1WBBPuA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "@smithy/util-uri-escape": "^3.0.0", "tslib": "^2.6.2" }, @@ -3312,12 +3789,12 @@ } }, "node_modules/@smithy/querystring-parser": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.8.tgz", - "integrity": "sha512-BtEk3FG7Ks64GAbt+JnKqwuobJNX8VmFLBsKIwWr1D60T426fGrV2L3YS5siOcUhhp6/Y6yhBw1PSPxA5p7qGg==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.10.tgz", + "integrity": "sha512-Oa0XDcpo9SmjhiDD9ua2UyM3uU01ZTuIrNdZvzwUTykW1PM8o2yJvMh1Do1rY5sUQg4NDV70dMi0JhDx4GyxuQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3325,24 +3802,24 @@ } }, "node_modules/@smithy/service-error-classification": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.8.tgz", - "integrity": "sha512-uEC/kCCFto83bz5ZzapcrgGqHOh/0r69sZ2ZuHlgoD5kYgXJEThCoTuw/y1Ub3cE7aaKdznb+jD9xRPIfIwD7g==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.10.tgz", + "integrity": "sha512-zHe642KCqDxXLuhs6xmHVgRwy078RfqxP2wRDpIyiF8EmsWXptMwnMwbVa50lw+WOGNrYm9zbaEg0oDe3PTtvQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0" + "@smithy/types": "^3.7.1" }, "engines": { "node": ">=16.0.0" } }, "node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.9.tgz", - "integrity": "sha512-/+OsJRNtoRbtsX0UpSgWVxFZLsJHo/4sTr+kBg/J78sr7iC+tHeOvOJrS5hCpVQ6sWBbhWLp1UNiuMyZhE6pmA==", + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3350,16 +3827,16 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.1.tgz", - "integrity": "sha512-NsV1jF4EvmO5wqmaSzlnTVetemBS3FZHdyc5CExbDljcyJCEEkJr8ANu2JvtNbVg/9MvKAWV44kTrGS+Pi4INg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^3.0.0", - "@smithy/protocol-http": "^4.1.5", - "@smithy/types": "^3.6.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.8", + "@smithy/util-middleware": "^3.0.10", "@smithy/util-uri-escape": "^3.0.0", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" @@ -3369,17 +3846,17 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.2.tgz", - "integrity": "sha512-dxw1BDxJiY9/zI3cBqfVrInij6ShjpV4fmGHesGZZUiP9OSE/EVfdwdRz0PgvkEvrZHpsj2htRaHJfftE8giBA==", + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.5.tgz", + "integrity": "sha512-k0sybYT9zlP79sIKd1XGm4TmK0AS1nA2bzDHXx7m0nGi3RQ8dxxQUs4CPkSmQTKAo+KF9aINU3KzpGIpV7UoMw==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^2.5.1", - "@smithy/middleware-endpoint": "^3.2.1", - "@smithy/middleware-stack": "^3.0.8", - "@smithy/protocol-http": "^4.1.5", - "@smithy/types": "^3.6.0", - "@smithy/util-stream": "^3.2.1", + "@smithy/core": "^2.5.4", + "@smithy/middleware-endpoint": "^3.2.4", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", "tslib": "^2.6.2" }, "engines": { @@ -3387,9 +3864,9 @@ } }, "node_modules/@smithy/types": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.6.0.tgz", - "integrity": "sha512-8VXK/KzOHefoC65yRgCn5vG1cysPJjHnOVt9d0ybFQSmJgQj152vMn4EkYhGuaOmnnZvCPav/KnYyE6/KsNZ2w==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -3399,13 +3876,13 @@ } }, "node_modules/@smithy/url-parser": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.8.tgz", - "integrity": "sha512-4FdOhwpTW7jtSFWm7SpfLGKIBC9ZaTKG5nBF0wK24aoQKQyDIKUw3+KFWCQ9maMzrgTJIuOvOnsV2lLGW5XjTg==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.10.tgz", + "integrity": "sha512-j90NUalTSBR2NaZTuruEgavSdh8MLirf58LoGSk4AtQfyIymogIhgnGUU2Mga2bkMkpSoC9gxb74xBXL5afKAQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/querystring-parser": "^3.0.8", - "@smithy/types": "^3.6.0", + "@smithy/querystring-parser": "^3.0.10", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" } }, @@ -3470,14 +3947,14 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "3.0.25", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.25.tgz", - "integrity": "sha512-fRw7zymjIDt6XxIsLwfJfYUfbGoO9CmCJk6rjJ/X5cd20+d2Is7xjU5Kt/AiDt6hX8DAf5dztmfP5O82gR9emA==", + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.28.tgz", + "integrity": "sha512-6bzwAbZpHRFVJsOztmov5PGDmJYsbNSoIEfHSJJyFLzfBGCCChiO3od9k7E/TLgrCsIifdAbB9nqbVbyE7wRUw==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^3.1.8", - "@smithy/smithy-client": "^3.4.2", - "@smithy/types": "^3.6.0", + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", "bowser": "^2.11.0", "tslib": "^2.6.2" }, @@ -3486,17 +3963,17 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "3.0.25", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.25.tgz", - "integrity": "sha512-H3BSZdBDiVZGzt8TG51Pd2FvFO0PAx/A0mJ0EH8a13KJ6iUCdYnw/Dk/MdC1kTd0eUuUGisDFaxXVXo4HHFL1g==", + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.28.tgz", + "integrity": "sha512-78ENJDorV1CjOQselGmm3+z7Yqjj5HWCbjzh0Ixuq736dh1oEnD9sAttSBNSLlpZsX8VQnmERqA2fEFlmqWn8w==", "license": "Apache-2.0", "dependencies": { - "@smithy/config-resolver": "^3.0.10", - "@smithy/credential-provider-imds": "^3.2.5", - "@smithy/node-config-provider": "^3.1.9", - "@smithy/property-provider": "^3.1.8", - "@smithy/smithy-client": "^3.4.2", - "@smithy/types": "^3.6.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/credential-provider-imds": "^3.2.7", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3504,13 +3981,13 @@ } }, "node_modules/@smithy/util-endpoints": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.1.4.tgz", - "integrity": "sha512-kPt8j4emm7rdMWQyL0F89o92q10gvCUa6sBkBtDJ7nV2+P7wpXczzOfoDJ49CKXe5CCqb8dc1W+ZdLlrKzSAnQ==", + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.1.6.tgz", + "integrity": "sha512-mFV1t3ndBh0yZOJgWxO9J/4cHZVn5UG1D8DeCc6/echfNkeEJWu9LD7mgGH5fHrEdR7LDoWw7PQO6QiGpHXhgA==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^3.1.9", - "@smithy/types": "^3.6.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3530,12 +4007,12 @@ } }, "node_modules/@smithy/util-middleware": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.8.tgz", - "integrity": "sha512-p7iYAPaQjoeM+AKABpYWeDdtwQNxasr4aXQEA/OmbOaug9V0odRVDy3Wx4ci8soljE/JXQo+abV0qZpW8NX0yA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^3.6.0", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3543,13 +4020,13 @@ } }, "node_modules/@smithy/util-retry": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.8.tgz", - "integrity": "sha512-TCEhLnY581YJ+g1x0hapPz13JFqzmh/pMWL2KEFASC51qCfw3+Y47MrTmea4bUE5vsdxQ4F6/KFbUeSz22Q1ow==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.10.tgz", + "integrity": "sha512-1l4qatFp4PiU6j7UsbasUHL2VU023NRB/gfaa1M0rDqVrRN4g3mCArLRyH3OuktApA4ye+yjWQHjdziunw2eWA==", "license": "Apache-2.0", "dependencies": { - "@smithy/service-error-classification": "^3.0.8", - "@smithy/types": "^3.6.0", + "@smithy/service-error-classification": "^3.0.10", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3557,14 +4034,14 @@ } }, "node_modules/@smithy/util-stream": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.2.1.tgz", - "integrity": "sha512-R3ufuzJRxSJbE58K9AEnL/uSZyVdHzud9wLS8tIbXclxKzoe09CRohj2xV8wpx5tj7ZbiJaKYcutMm1eYgz/0A==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", "license": "Apache-2.0", "dependencies": { - "@smithy/fetch-http-handler": "^4.0.0", - "@smithy/node-http-handler": "^3.2.5", - "@smithy/types": "^3.6.0", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", "@smithy/util-base64": "^3.0.0", "@smithy/util-buffer-from": "^3.0.0", "@smithy/util-hex-encoding": "^3.0.0", @@ -3601,13 +4078,13 @@ } }, "node_modules/@smithy/util-waiter": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.7.tgz", - "integrity": "sha512-d5yGlQtmN/z5eoTtIYgkvOw27US2Ous4VycnXatyoImIF9tzlcpnKqQ/V7qhvJmb2p6xZne1NopCLakdTnkBBQ==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.9.tgz", + "integrity": "sha512-/aMXPANhMOlMPjfPtSrDfPeVP8l56SJlz93xeiLmhLe5xvlXA5T3abZ2ilEsDEPeY9T/wnN/vNGn9wa1SbufWA==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^3.1.6", - "@smithy/types": "^3.6.0", + "@smithy/abort-controller": "^3.1.8", + "@smithy/types": "^3.7.1", "tslib": "^2.6.2" }, "engines": { @@ -3786,12 +4263,12 @@ } }, "node_modules/@types/node": { - "version": "22.4.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.4.1.tgz", - "integrity": "sha512-1tbpb9325+gPnKK0dMm+/LMriX0vKxf6RnB0SZUqfyVkQ4fMgUSySqhxE/y8Jvs4NyF1yHzTfG9KlnkIODxPKg==", + "version": "22.10.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.0.tgz", + "integrity": "sha512-XC70cRZVElFHfIUB40FgZOBbgJYFKKMa5nb9lxcwYstFG/Mi+/Y0bGS+rs6Dmhmkpq4pnNiLiuZAbc02YCOnmA==", "license": "MIT", "dependencies": { - "undici-types": "~6.19.2" + "undici-types": "~6.20.0" } }, "node_modules/@types/pg": { @@ -3849,14 +4326,14 @@ "license": "MIT" }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.14.0.tgz", - "integrity": "sha512-aBbBrnW9ARIDn92Zbo7rguLnqQ/pOrUguVpbUwzOhkFg2npFDwTgPGqFqE0H5feXcOoJOfX3SxlJaKEVtq54dw==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.16.0.tgz", + "integrity": "sha512-mwsZWubQvBki2t5565uxF0EYvG+FwdFb8bMtDuGQLdCCnGPrDEDvm1gtfynuKlnpzeBRqdFCkMf9jg1fnAK8sg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.14.0", - "@typescript-eslint/visitor-keys": "8.14.0" + "@typescript-eslint/types": "8.16.0", + "@typescript-eslint/visitor-keys": "8.16.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3867,9 +4344,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.14.0.tgz", - "integrity": "sha512-yjeB9fnO/opvLJFAsPNYlKPnEM8+z4og09Pk504dkqonT02AyL5Z9SSqlE0XqezS93v6CXn49VHvB2G7XSsl0g==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.16.0.tgz", + "integrity": "sha512-NzrHj6thBAOSE4d9bsuRNMvk+BvaQvmY4dDglgkgGC0EW/tB3Kelnp3tAKH87GEwzoxgeQn9fNGRyFJM/xd+GQ==", "dev": true, "license": "MIT", "engines": { @@ -3881,14 +4358,14 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.14.0.tgz", - "integrity": "sha512-OPXPLYKGZi9XS/49rdaCbR5j/S14HazviBlUQFvSKz3npr3NikF+mrgK7CFVur6XEt95DZp/cmke9d5i3vtVnQ==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.16.0.tgz", + "integrity": "sha512-E2+9IzzXMc1iaBy9zmo+UYvluE3TW7bCGWSF41hVWUE01o8nzr1rvOQYSxelxr6StUvRcTMe633eY8mXASMaNw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/types": "8.14.0", - "@typescript-eslint/visitor-keys": "8.14.0", + "@typescript-eslint/types": "8.16.0", + "@typescript-eslint/visitor-keys": "8.16.0", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -3961,16 +4438,16 @@ "license": "MIT" }, "node_modules/@typescript-eslint/utils": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.14.0.tgz", - "integrity": "sha512-OGqj6uB8THhrHj0Fk27DcHPojW7zKwKkPmHXHvQ58pLYp4hy8CSUdTKykKeh+5vFqTTVmjz0zCOOPKRovdsgHA==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.16.0.tgz", + "integrity": "sha512-C1zRy/mOL8Pj157GiX4kaw7iyRLKfJXBR3L82hk5kS/GyHcOFmy4YUq/zfZti72I9wnuQtA/+xzft4wCC8PJdA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.14.0", - "@typescript-eslint/types": "8.14.0", - "@typescript-eslint/typescript-estree": "8.14.0" + "@typescript-eslint/scope-manager": "8.16.0", + "@typescript-eslint/types": "8.16.0", + "@typescript-eslint/typescript-estree": "8.16.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3981,17 +4458,22 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.14.0.tgz", - "integrity": "sha512-vG0XZo8AdTH9OE6VFRwAZldNc7qtJ/6NLGWak+BtENuEUXGZgFpihILPiBvKXvJ2nFu27XNGC6rKiwuaoMbYzQ==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.16.0.tgz", + "integrity": "sha512-pq19gbaMOmFE3CbL0ZB8J8BFCo2ckfHBfaIsaOZgBIF4EoISJIdLX5xRhd0FGB0LlHReNRuzoJoMGpTjq8F2CQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.14.0", - "eslint-visitor-keys": "^3.4.3" + "@typescript-eslint/types": "8.16.0", + "eslint-visitor-keys": "^4.2.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4001,6 +4483,19 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", + "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", @@ -4923,9 +5418,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001680", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001680.tgz", - "integrity": "sha512-rPQy70G6AGUMnbwS1z6Xg+RkHYPAi18ihs47GH0jcxIG7wArmPgY3XbS2sRdBbxJljp3thdT8BIqv9ccCypiPA==", + "version": "1.0.30001684", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001684.tgz", + "integrity": "sha512-G1LRwLIQjBQoyq0ZJGqGIJUXzJ8irpbjHLpVRXDvBEScFJ9b17sgK6vlx0GAJFE21okD7zXl08rRRUfq6HdoEQ==", "dev": true, "funding": [ { @@ -5260,9 +5755,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz", - "integrity": "sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -5521,9 +6016,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.56", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.56.tgz", - "integrity": "sha512-7lXb9dAvimCFdvUMTyucD4mnIndt/xhRKFAlky0CyFogdnNmdPQNoHI23msF/2V4mpTxMzgMdjK4+YRlFlRQZw==", + "version": "1.5.67", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.67.tgz", + "integrity": "sha512-nz88NNBsD7kQSAGGJyp8hS6xSPtWwqNogA0mjtc2nUYeEf3nURK9qpV18TuBdDmEDgVWotS8Wkzf+V52dSQ/LQ==", "dev": true, "license": "ISC" }, @@ -6299,9 +6794,9 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz", + "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", "dev": true, "license": "ISC" }, @@ -6704,9 +7199,9 @@ } }, "node_modules/google-auth-library": { - "version": "9.14.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.14.2.tgz", - "integrity": "sha512-R+FRIfk1GBo3RdlRYWPdwk8nmtVUOn6+BkDomAC46KoU8kzXzE1HLmOasSCbWUByMMAGkknVF0G5kQ69Vj7dlA==", + "version": "9.15.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.0.tgz", + "integrity": "sha512-7ccSEJFDFO7exFbO6NRyC+xH8/mZ1GZGG2xxx9iHxZWcjUjJpjWxIMw3cofAKcueZ6DATiukmmprD7yavQHOyQ==", "license": "Apache-2.0", "dependencies": { "base64-js": "^1.3.0", @@ -6721,12 +7216,15 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.1.0.tgz", + "integrity": "sha512-FQoVQnqcdk4hVM4JN1eromaun4iuS34oStkdlLENLdpULsuQcTyXj8w7ayhuUfPwEYZ1ZOooOTT6fdA9Vmx/RA==", "license": "MIT", "dependencies": { - "get-intrinsic": "^1.1.3" + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -9210,9 +9708,9 @@ } }, "node_modules/node-addon-api": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.2.tgz", - "integrity": "sha512-9emqXAKhVoNrQ792nLI/wpzPpJ/bj/YXxW0CvAau1+RdGBcCRF1Dmz7719zgVsQNrzHl9Tzn3ImZ4qWFarWL0A==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.3.0.tgz", + "integrity": "sha512-8VOpLHFrOQlAH+qA0ZzuGRlALRA6/LVh8QJldbrC4DY0hXoMP0l4Acq8TzFC018HztWiRqyCEj2aTWY2UvnJUg==", "license": "MIT", "engines": { "node": "^18 || ^20 || >= 21" @@ -9264,9 +9762,9 @@ } }, "node_modules/node-gyp-build": { - "version": "4.8.2", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.2.tgz", - "integrity": "sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==", + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", "license": "MIT", "bin": { "node-gyp-build": "bin.js", @@ -9381,9 +9879,9 @@ "license": "MIT" }, "node_modules/node-rdkafka": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/node-rdkafka/-/node-rdkafka-3.1.1.tgz", - "integrity": "sha512-3rY24KlFvkQ0Pfqo5HP/fXa1GA+8R7hFLP+rm2htT7XJluH2lJ1fcEM5KDWKpkq1Nf4otHYvnBdY5eRH6ecYjg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/node-rdkafka/-/node-rdkafka-3.2.0.tgz", + "integrity": "sha512-Xt30tiwTKv6LXLtelNDXYETb1VnrzPU5s5OAeyY6fz20Bpqy5ARDoWKim4zsKugGlaztsmzPauqMYcQrLdk8HQ==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -11989,9 +12487,9 @@ "license": "MIT" }, "node_modules/ts-api-utils": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.0.tgz", - "integrity": "sha512-032cPxaEKwM+GT3vA5JXNzIaizx388rhsSW79vGRNGXfRRAdEAn2mvk36PvK5HnOchyWZ7afLEXqYCvPCrzuzQ==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", "dev": true, "license": "MIT", "engines": { @@ -12070,9 +12568,9 @@ } }, "node_modules/typescript": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", - "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", + "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -12083,9 +12581,9 @@ } }, "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", "license": "MIT" }, "node_modules/unique-filename": { @@ -12314,9 +12812,9 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", - "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.16.tgz", + "integrity": "sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==", "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", @@ -12436,13 +12934,16 @@ } }, "node_modules/wtfnode": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/wtfnode/-/wtfnode-0.9.3.tgz", - "integrity": "sha512-MXjgxJovNVYUkD85JBZTKT5S5ng/e56sNuRZlid7HcGTNrIODa5UPtqE3i0daj7fJ2SGj5Um2VmiphQVyVKK5A==", + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/wtfnode/-/wtfnode-0.9.4.tgz", + "integrity": "sha512-5xgeLjIxZ8DVHU4ty3kOdd9QfHDxf89tmSy0+yN8n59S3wx6LBJh8XhEg61OPOGE65jEYGAtLq0GMzLKrsjfPQ==", "dev": true, "license": "ISC", "bin": { "wtfnode": "proxy.js" + }, + "engines": { + "node": ">=0.10.0" } }, "node_modules/xml2js": { @@ -12494,9 +12995,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.0.tgz", - "integrity": "sha512-a6ae//JvKDEra2kdi1qzCyrJW/WZCgFi8ydDV+eXExl95t+5R+ijnqHJbz9tmMh8FUjx3iv2fCQ4dclAQlO2UQ==", + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz", + "integrity": "sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==", "license": "ISC", "bin": { "yaml": "bin.mjs" @@ -12587,12 +13088,21 @@ } }, "node_modules/yazl": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/yazl/-/yazl-2.5.1.tgz", - "integrity": "sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", "license": "MIT", "dependencies": { - "buffer-crc32": "~0.2.3" + "buffer-crc32": "^1.0.0" + } + }, + "node_modules/yazl/node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" } }, "node_modules/yocto-queue": { diff --git a/package.json b/package.json index 4e16fa1777..e4bdec37d0 100644 --- a/package.json +++ b/package.json @@ -72,12 +72,12 @@ }, "dependencies": { "@aws-sdk/client-s3": "3.689.0", - "@aws-sdk/client-sts": "3.687.0", + "@aws-sdk/client-sts": "3.699.0", "@azure/identity": "4.5.0", "@azure/monitor-query": "1.3.1", "@azure/storage-blob": "12.25.0", "@google-cloud/storage": "7.14.0", - "@smithy/node-http-handler": "3.2.5", + "@smithy/node-http-handler": "3.3.1", "ajv": "8.17.1", "aws-sdk": "2.1692.0", "bcrypt": "5.1.1", @@ -107,8 +107,8 @@ "morgan": "1.10.0", "nan": "2.22.0", "ncp": "2.0.0", - "node-addon-api": "8.2.2", - "node-rdkafka": "3.1.1", + "node-addon-api": "8.3.0", + "node-rdkafka": "3.2.0", "performance-now": "2.1.0", "pg": "8.13.1", "ping": "0.4.4", @@ -117,14 +117,14 @@ "rimraf": "6.0.1", "seedrandom": "3.0.5", "setimmediate": "1.0.5", - "typescript": "5.6.3", + "typescript": "5.7.2", "utf-8-validate": "6.0.5", "uuid": "10.0.0", "ws": "8.18.0", "xml2js": "0.6.2", - "yaml": "2.6.0", + "yaml": "2.6.1", "yauzl": "3.2.0", - "yazl": "2.5.1" + "yazl": "3.3.1" }, "devDependencies": { "@aws-sdk/lib-storage": "3.689.0", @@ -132,9 +132,8 @@ "@types/jest": "29.5.14", "@types/lodash": "4.17.13", "@types/mongodb": "4.0.7", - "@types/node": "22.4.1", + "@types/node": "22.10.0", "@types/pg": "8.11.10", - "@types/request": "2.48.12", "eslint": "8.57.1", "eslint-plugin-header": "3.1.1", "eslint-plugin-jest": "28.9.0", @@ -144,6 +143,6 @@ "pkg": "5.8.1", "pkg-fetch": "3.5.2", "sinon": "19.0.2", - "wtfnode": "0.9.3" + "wtfnode": "0.9.4" } } From 59b917f83915cc01994bb99caa7bc5483b362847 Mon Sep 17 00:00:00 2001 From: jackyalbo Date: Wed, 4 Dec 2024 17:25:24 +0200 Subject: [PATCH 3/6] minor fixes to md_blow Signed-off-by: jackyalbo (cherry picked from commit da6d28ce54a9dea657ec59dd8b1603b275eb3985) --- src/tools/md_blow.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tools/md_blow.js b/src/tools/md_blow.js index 6a39005b3c..ab50977706 100644 --- a/src/tools/md_blow.js +++ b/src/tools/md_blow.js @@ -7,22 +7,23 @@ const crypto = require('crypto'); const P = require('../util/promise'); const api = require('../api'); const dbg = require('../util/debug_module')(__filename); -const system_store = require('../server/system_services/system_store').get_instance(); +const system_store = require('../server/system_services/system_store').get_instance({ standalone: true }); const node_allocator = require('../server/node_services/node_allocator'); const db_client = require('../util/db_client'); const rpc = api.new_rpc(); const client = rpc.new_client(); -argv.email = argv.email || 'demo@noobaa.com'; +argv.email = argv.email || 'admin@noobaa.io'; argv.password = argv.password || 'DeMo1'; -argv.system = argv.system || 'demo'; +argv.system = argv.system || 'noobaa'; argv.bucket = argv.bucket || 'first.bucket'; argv.count = argv.count || 100; argv.chunks = argv.chunks || 128; argv.chunk_size = argv.chunk_size || 1024 * 1024; argv.concur = argv.concur || 20; argv.key = argv.key || ('md_blow-' + Date.now().toString(36)); +argv.pool = argv.pool || 'noobaa-default-backing-store'; main(); @@ -79,7 +80,7 @@ async function blow_parts(params) { const [record] = bucket.tiering.tiers; const tier = await client.tier.read_tier({ name: record.tier }); const tier_db = _.find(system_store.data.tiers, t => (t.name.unwrap() === tier.name.unwrap())); - const pool_db = system_store.data.pools[0]; + const pool_db = _.find(system_store.data.pools, p => (p.name === argv.pool)); await node_allocator.refresh_pool_alloc(pool_db); const node = node_allocator.allocate_node({ pools: [pool_db] From 737f89fc18e43880d0be292f3f260dec614159a7 Mon Sep 17 00:00:00 2001 From: shirady <57721533+shirady@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:31:57 +0200 Subject: [PATCH 4/6] NC | NSFS | Change printing of `Warning stuck buffer_pool buffer` from `console.warn` to `console.error` Signed-off-by: shirady <57721533+shirady@users.noreply.github.com> (cherry picked from commit 43849a3c655ef1981c00bc80d772b9df4432e0b5) --- src/util/buffer_utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/buffer_utils.js b/src/util/buffer_utils.js index 34845f1b67..25576a03ae 100644 --- a/src/util/buffer_utils.js +++ b/src/util/buffer_utils.js @@ -217,7 +217,7 @@ class BuffersPool { if (this.warning_timeout) { const err = new Error('Warning stuck buffer_pool buffer'); warning_timer = setTimeout(() => { - console.error(err.stack); + console.warn(err.stack); }, this.warning_timeout); warning_timer.unref(); } From 6f52277d526858688a553ff9c5d5b736e669ee7e Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:41:37 +0200 Subject: [PATCH 5/6] NC | Online Upgrade | Health CLI update config directory and upgrade checks Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> (cherry picked from commit eaf443fb5c5b5cfac8fac0c07d8170e75a6d3438) --- docs/NooBaaNonContainerized/CI&Tests.md | 2 +- docs/NooBaaNonContainerized/Health.md | 38 +- src/manage_nsfs/health.js | 90 +++- src/test/unit_tests/nc_index.js | 2 +- src/test/unit_tests/sudo_index.js | 2 +- ...st_nc_nsfs_health.js => test_nc_health.js} | 406 +++++++++++------- src/upgrade/nc_upgrade_manager.js | 3 +- src/upgrade/upgrade_utils.js | 4 +- 8 files changed, 365 insertions(+), 182 deletions(-) rename src/test/unit_tests/{test_nc_nsfs_health.js => test_nc_health.js} (68%) diff --git a/docs/NooBaaNonContainerized/CI&Tests.md b/docs/NooBaaNonContainerized/CI&Tests.md index 1ebc40fda1..160cc8ebfd 100644 --- a/docs/NooBaaNonContainerized/CI&Tests.md +++ b/docs/NooBaaNonContainerized/CI&Tests.md @@ -88,7 +88,7 @@ Run `NC mocha tests` with root permissions - #### NC mocha tests The following is a list of `NC mocha test` files - 1. `test_nc_nsfs_cli.js` - Tests NooBaa CLI. -2. `test_nc_nsfs_health` - Tests NooBaa Health CLI. +2. `test_nc_health` - Tests NooBaa Health CLI. 3. `test_nsfs_glacier_backend.js` - Tests NooBaa Glacier Backend. 4. `test_nc_with_a_couple_of_forks.js` - Tests the `bucket_namespace_cache` when running with a couple of forks. Please notice that it uses `nc_coretest` with setup that includes a couple of forks. diff --git a/docs/NooBaaNonContainerized/Health.md b/docs/NooBaaNonContainerized/Health.md index d3184256c2..08bf0c981b 100644 --- a/docs/NooBaaNonContainerized/Health.md +++ b/docs/NooBaaNonContainerized/Health.md @@ -34,6 +34,14 @@ For more details about NooBaa RPM installation, see - [Getting Started](./Gettin - Iterating buckets under the config directory. - Confirming the existence of the bucket's configuration file and its validity as a JSON file. - Verifying that the underlying storage path of a bucket exists. + - `Config directory health` + - checks if config system and directory data exists + - returns the config directory status + - `Config directory upgrade health` + - checks if config system and directory data exists + - checks if there is ongoing upgrade + - returns error if there is no ongoing upgrade, but the config directory phase is locked + - returns message if there is no ongoing upgrade and the config directory is unlocked * Health CLI requires root permissions. @@ -148,6 +156,11 @@ The output of the Health CLI is a JSON object containing the following propertie - Enum: 'PERSISTENT' | 'TEMPORARY' - Description: For TEMPORARY error types, NooBaa attempts multiple retries before updating the status to reflect an error. Currently, TEMPORARY error types are only observed in checks for invalid NooBaa endpoints. +- `config_directory` + - Type: Object {"phase": "CONFIG_DIR_UNLOCKED" | "CONFIG_DIR_LOCKED","config_dir_version": String, + "upgrade_package_version": String, "upgrade_status": Object, "error": Object }. + - Description: An object that consists config directory information, config directory upgrade information etc. + - Example: { "phase": "CONFIG_DIR_UNLOCKED", "config_dir_version": "1.0.0", "upgrade_package_version": "5.18.0", "upgrade_status": { "message": "there is no in-progress upgrade" }} ## Example ```sh @@ -225,6 +238,14 @@ Output: } ], "error_type": "PERSISTENT" + }, + "config_directory": { + "phase": "CONFIG_DIR_UNLOCKED", + "config_dir_version": "1.0.0", + "upgrade_package_version": "5.18.0", + "upgrade_status": { + "message": "there is no in-progress upgrade" + } } } } @@ -243,7 +264,8 @@ Output: - The config file of bucket1 is invalid. Therefore, NooBaa health reports INVALID_CONFIG. - The underlying file system directory of bucket3 is missing. Therefore, NooBaa health reports STORAGE_NOT_EXIST. - +- config_directory: + - the config directory phase is unlocked, config directory version is "1.0.0", matching source code/package version is "5.18.0" and there is no ongoing upgrade. ## Health Errors @@ -365,4 +387,16 @@ The following error codes will be associated with a specific Bucket or Account s - Reasons: - Bucket missing owner account. - Resolutions: - - Check for owner_account property in bucket config file. \ No newline at end of file + - Check for owner_account property in bucket config file. + +#### 8. Config Directory is invalid + - Error code: `INVALID_CONFIG_DIR` + - Error message: Config directory is invalid + - Reasons: + - System.json is missing - NooBaa was never started + - Config directory property is missing in system.json - the user didn't run config directory upgrade when upgrading from 5.17.z to 5.18.0 + - Config directory upgrade error. + - Resolutions: + - Start NooBaa service + - Run `noobaa-cli upgrade` + - Check the in_progress_upgrade the exact reason for the failure. diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 25629b0273..65ccf24da3 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -14,6 +14,7 @@ const { TYPES } = require('./manage_nsfs_constants'); const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils'); const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; +const { CONFIG_DIR_LOCKED, CONFIG_DIR_UNLOCKED } = require('../upgrade/nc_upgrade_manager'); const HOSTNAME = 'localhost'; @@ -60,6 +61,10 @@ const health_errors = { error_code: 'MISSING_ACCOUNT_OWNER', error_message: 'Bucket account owner not found', }, + INVALID_CONFIG_DIR: { + error_code: 'INVALID_CONFIG_DIR', + error_message: 'Config directory is invalid', + }, UNKNOWN_ERROR: { error_code: 'UNKNOWN_ERROR', error_message: 'An unknown error occurred', @@ -117,12 +122,16 @@ class NSFSHealth { endpoint_state = await this.get_endpoint_response(); memory = await this.get_service_memory_usage(); } + // TODO: add more health status based on system.json, e.g. RPM upgrade issues + const system_data = await this.config_fs.get_system_config_file({ silent_if_missing: true }); + const config_directory_status = this._get_config_dir_status(system_data); + let bucket_details; let account_details; - const response_code = endpoint_state ? endpoint_state.response.response_code : 'NOT_RUNNING'; - const service_health = service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ? 'NOTOK' : 'OK'; - - const error_code = await this.get_error_code(service_status, pid, response_code); + const endpoint_response_code = (endpoint_state && endpoint_state.response?.response_code) || 'UNKNOWN_ERROR'; + const health_check_params = { service_status, pid, endpoint_response_code, config_directory_status }; + const service_health = this._calc_health_status(health_check_params); + const error_code = this.get_error_code(health_check_params); if (this.all_bucket_details) bucket_details = await this.get_bucket_status(); if (this.all_account_details) account_details = await this.get_account_status(); const health = { @@ -136,6 +145,7 @@ class NSFSHealth { endpoint_state, error_type: health_errors_tyes.TEMPORARY, }, + config_directory_status, accounts_status: { invalid_accounts: account_details === undefined ? undefined : account_details.invalid_storages, valid_accounts: account_details === undefined ? undefined : account_details.valid_storages, @@ -161,7 +171,7 @@ class NSFSHealth { delay_ms: config.NC_HEALTH_ENDPOINT_RETRY_DELAY, func: async () => { endpoint_state = await this.get_endpoint_fork_response(); - if (endpoint_state.response.response_code === fork_response_code.NOT_RUNNING.response_code) { + if (endpoint_state.response?.response_code === fork_response_code.NOT_RUNNING.response_code) { throw new Error('Noobaa endpoint is not running, all the retries failed'); } } @@ -173,13 +183,23 @@ class NSFSHealth { return endpoint_state; } - async get_error_code(nsfs_status, pid, endpoint_response_code) { - if (nsfs_status !== 'active' || pid === '0') { + /** + * get_error_code returns the error code per the failed check + * @param {{service_status: String, + * pid: string, + * endpoint_response_code: string, + * config_directory_status: Object }} health_check_params + * @returns {Object} + */ + get_error_code({ service_status, pid, endpoint_response_code, config_directory_status }) { + if (service_status !== 'active' || pid === '0') { return health_errors.NOOBAA_SERVICE_FAILED; } else if (endpoint_response_code === 'NOT_RUNNING') { return health_errors.NOOBAA_ENDPOINT_FAILED; } else if (endpoint_response_code === 'MISSING_FORKS') { return health_errors.NOOBAA_ENDPOINT_FORK_MISSING; + } else if (config_directory_status.error) { + return health_errors.CONFIG_DIR_ERROR; } } @@ -239,7 +259,7 @@ class NSFSHealth { const fork_count_response = await this.make_endpoint_health_request(url_path); if (!fork_count_response) { return { - response_code: fork_response_code.NOT_RUNNING, + response: fork_response_code.NOT_RUNNING, total_fork_count: total_fork_count, running_workers: worker_ids, }; @@ -421,6 +441,60 @@ class NSFSHealth { err_obj }; } + + /** + * _get_config_dir_status returns the config directory phase, version, + * matching package_version, upgrade_status and error if occured. + * @param {Object} system_data + * @returns {Object} + */ + _get_config_dir_status(system_data) { + if (!system_data) return { error: 'system data is missing' }; + const config_dir_data = system_data.config_directory; + if (!config_dir_data) return { error: 'config directory data is missing, must upgrade config directory' }; + const config_dir_upgrade_status = this._get_config_dir_upgrade_status(config_dir_data); + return { + phase: config_dir_data.phase, + config_dir_version: config_dir_data.config_dir_version, + upgrade_package_version: config_dir_data.upgrade_package_version, + upgrade_status: config_dir_upgrade_status, + error: config_dir_upgrade_status.error || undefined + }; + } + + /** + * _get_config_dir_upgrade_status returns one of the following + * 1. the status of an ongoing upgrade, if valid it returns an object with upgrade details + * 2. if upgrade is not ongoing but config dir is locked, the error details of the upgrade's last_failure will return + * 3. if upgrade is not ongoing and config dir is unlocked, a corresponding message will return + * @param {Object} config_dir_data + * @returns {Object} + */ + _get_config_dir_upgrade_status(config_dir_data) { + if (config_dir_data.in_progress_upgrade) return { in_progress_upgrade: config_dir_data.in_progress_upgrade }; + if (config_dir_data.phase === CONFIG_DIR_LOCKED) { + return { error: 'last_upgrade_failed', last_failure: config_dir_data.upgrade_history.last_failure }; + } + if (config_dir_data.phase === CONFIG_DIR_UNLOCKED) { + return { message: 'there is no in-progress upgrade' }; + } + } + + /** + * _calc_health_status calcs the overall health status of NooBaa NC + * @param {{service_status: String, + * pid: string, + * endpoint_response_code: string, + * config_directory_status: Object }} health_check_params + * @returns {'OK' | 'NOTOK'} + */ + _calc_health_status({ service_status, pid, endpoint_response_code, config_directory_status }) { + const is_unhealthy = service_status !== 'active' || + pid === '0' || + endpoint_response_code !== 'RUNNING' || + config_directory_status.error; + return is_unhealthy ? 'NOTOK' : 'OK'; + } } async function get_health_status(argv, config_fs) { diff --git a/src/test/unit_tests/nc_index.js b/src/test/unit_tests/nc_index.js index 5497e05b9b..49cc875263 100644 --- a/src/test/unit_tests/nc_index.js +++ b/src/test/unit_tests/nc_index.js @@ -11,7 +11,7 @@ require('./test_chunk_fs'); require('./test_namespace_fs_mpu'); require('./test_nb_native_fs'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); require('./test_nsfs_access'); require('./test_nsfs_integration'); require('./test_bucketspace_fs'); diff --git a/src/test/unit_tests/sudo_index.js b/src/test/unit_tests/sudo_index.js index fa3017e028..37068a51d6 100644 --- a/src/test/unit_tests/sudo_index.js +++ b/src/test/unit_tests/sudo_index.js @@ -21,5 +21,5 @@ require('./test_bucketspace_versioning'); require('./test_bucketspace_fs'); require('./test_nsfs_versioning'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_health.js similarity index 68% rename from src/test/unit_tests/test_nc_nsfs_health.js rename to src/test/unit_tests/test_nc_health.js index e8100a8f60..5d4889335b 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_health.js @@ -1,5 +1,5 @@ /* Copyright (C) 2016 NooBaa */ -/*eslint max-lines-per-function: ["error", 700]*/ +/*eslint max-lines-per-function: ['error', 700]*/ 'use strict'; @@ -8,6 +8,7 @@ const mocha = require('mocha'); const sinon = require('sinon'); const assert = require('assert'); const config = require('../../../config'); +const pkg = require('../../../package.json'); const fs_utils = require('../../util/fs_utils'); const nb_native = require('../../util/nb_native'); const { ConfigFS } = require('../../sdk/config_fs'); @@ -18,11 +19,30 @@ const { get_process_fs_context } = require('../../util/native_fs_utils'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli } = require('../system_tests/test_utils'); +const { CONFIG_DIR_UNLOCKED, CONFIG_DIR_LOCKED } = require('../../upgrade/nc_upgrade_manager'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_health'); const DEFAULT_FS_CONFIG = get_process_fs_context(); const bucket_storage_path = path.join(tmp_fs_path, 'bucket_storage_path'); +const os = require('os'); +const hostname = os.hostname(); + +const valid_system_json = { + [hostname]: { + 'current_version': pkg.version, + 'upgrade_history': { 'successful_upgrades': [] }, + }, +}; + +const get_service_state_mock_default_response = [{ service_status: 'active', pid: 1000 }, { service_status: 'active', pid: 2000 }]; +const get_endpoint_response_mock_default_response = [{ response: { response_code: 'RUNNING', total_fork_count: 0 } }]; +const get_system_config_mock_default_response = [{ + ...valid_system_json, config_directory: { + phase: CONFIG_DIR_UNLOCKED, config_dir_version: '1.0.0', + package_version: pkg.version, upgrade_history: [] +} }]; +const default_mock_upgrade_status = { message: 'there is no in-progress upgrade' }; mocha.describe('nsfs nc health', function() { @@ -45,6 +65,10 @@ mocha.describe('nsfs nc health', function() { }); mocha.describe('nsfs nc health cli validations', function() { + mocha.afterEach(() => { + restore_health_if_needed(Health); + }); + mocha.it('https_port flag type validation - should fail', async function() { try { await exec_manage_cli(TYPES.DIAGNOSE, DIAGNOSE_ACTIONS.HEALTH, { 'http_port': '' }); @@ -137,8 +161,7 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, {config_root, ...account1_options}); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket1_options}); await fs_utils.file_must_exist(path.join(config_root, 'master_keys.json')); - const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage"); - get_service_memory_usage.onFirstCall().returns(Promise.resolve(100)); + set_mock_functions(Health, { get_service_memory_usage: [100]}); for (const user of Object.values(fs_users)) { await create_fs_user_by_platform(user.distinguished_name, user.distinguished_name, user.uid, user.gid); } @@ -158,62 +181,64 @@ mocha.describe('nsfs nc health', function() { mocha.afterEach(async () => { await fs_utils.file_delete(config_fs.config_json_path); + restore_health_if_needed(Health); }); mocha.it('Health all condition is success', async function() { - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_UNLOCKED + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + }); Health.all_account_details = true; Health.all_bucket_details = true; const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 0); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts[0].name, 'account1'); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.valid_buckets[0].name, 'bucket1'); + assert_config_dir_status(health_status, valid_system_json.config_directory); }); mocha.it('NooBaa service is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'inactive', pid: 0 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: [{ service_status: 'inactive', pid: 0 }], + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); assert.strictEqual(health_status.error.error_code, 'NOOBAA_SERVICE_FAILED'); }); mocha.it('NooBaa endpoint return error response is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'MISSING_FORKS', total_fork_count: 3, running_workers: ['1', '3']}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: [{ response: { response_code: 'MISSING_FORKS', total_fork_count: 3, running_workers: ['1', '3'] } }], + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); assert.strictEqual(health_status.error.error_code, 'NOOBAA_ENDPOINT_FORK_MISSING'); }); mocha.it('NSFS account with invalid storage path', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip invalid storage path check on the CLI const account_invalid_options = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid', nsfs_account_config: { new_buckets_path: path.join(new_buckets_path, '/invalid') } }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_options); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); @@ -223,23 +248,21 @@ mocha.describe('nsfs nc health', function() { mocha.it('NSFS bucket with invalid storage path', async function() { this.timeout(5000);// eslint-disable-line no-invalid-this - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); const resp = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account2_options }); const parsed_res = JSON.parse(resp).response.reply; // create it manually because we can not skip invalid storage path check on the CLI const bucket_invalid = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid', owner_account: parsed_res._id }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_invalid.name); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "STORAGE_NOT_EXIST"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'STORAGE_NOT_EXIST'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_invalid.name}); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account2_options.name}); }); @@ -250,22 +273,20 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_inaccessible_options.name); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_inaccessible_options.name}); @@ -277,18 +298,16 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with invalid owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account', owner_account: 'invalid_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "INVALID_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'INVALID_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); @@ -299,35 +318,30 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with empty owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "MISSING_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'MISSING_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); }); mocha.it('NSFS invalid bucket schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const bucket_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_schema', path: new_buckets_path }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_schema, 'invalid'); - - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); @@ -337,16 +351,14 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NSFS invalid account schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const account_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid_schema', path: new_buckets_path, bla: 5 }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_schema, 'invalid'); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); @@ -356,15 +368,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_account_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = false; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 0); @@ -374,15 +384,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_bucket_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status, undefined); @@ -392,18 +400,16 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Config root path without bucket and account folders', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; Health.config_root = config_root_invalid; const old_config_fs = Health.config_fs; Health.config_fs = new ConfigFS(config_root_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 0); @@ -419,20 +425,18 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_options.name}); }); @@ -441,15 +445,13 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, debug: 5, ...account_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_ACCESS_CHECK = false; @@ -466,15 +468,13 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_HEALTH_ACCESS_CHECK = false; @@ -490,36 +490,32 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_dn_options.name }); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_dn_options.name); }); mocha.it('Account with inaccessible path - dn - NC_DISABLE_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_ACCESS_CHECK = false; @@ -535,15 +531,13 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn - NC_DISABLE_HEALTH_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_HEALTH_ACCESS_CHECK = false; @@ -559,15 +553,13 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with invalid dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...invalid_account_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: invalid_account_dn_options.name }); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); @@ -576,21 +568,19 @@ mocha.describe('nsfs nc health', function() { const found = health_status.checks.accounts_status.invalid_accounts.filter(account => account.name === invalid_account_dn_options.name); assert.strictEqual(found.length, 1); - assert.strictEqual(found[0].code, "INVALID_DISTINGUISHED_NAME"); + assert.strictEqual(found[0].code, 'INVALID_DISTINGUISHED_NAME'); }); mocha.it('Account with new_buckets_path missing and allow_bucket_creation false, valid account', async function() { const account_valid = { name: 'account_valid', uid: 999, gid: 999, allow_bucket_creation: false }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_valid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_valid.name }); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); @@ -601,19 +591,107 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with new_buckets_path missing and allow_bucket_creation true, invalid account', async function() { const account_invalid = { name: 'account_invalid', uid: 999, gid: 999, allow_bucket_creation: true }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_invalid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_invalid.name }); }); + + mocha.it('Health all condition - failed config directory upgrade status', async function() { + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + last_failure: { error: 'mock error'} + } + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); + assert_config_dir_status(health_status, valid_system_json.config_directory); + }); + + mocha.it('Health all condition - valid ongoing config directory upgrade', async function() { + valid_system_json.config_directory = { + config_dir_version: config_fs.config_dir_version, + upgrade_package_version: pkg.version, + phase: CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + }, + in_progress_upgrade: { from_version: '5.17.8', to_version: '5.18.0'} + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); + assert_config_dir_status(health_status, valid_system_json.config_directory); + }); }); }); + +/** + * assert_config_dir_status asserts config directory status + * @param {Object} health_status + * @param {Object} expected_config_dir + */ +function assert_config_dir_status(health_status, expected_config_dir) { + const actual_config_dir_health = health_status.checks.config_directory_status; + assert.strictEqual(actual_config_dir_health.phase, expected_config_dir.phase); + assert.strictEqual(actual_config_dir_health.config_dir_version, expected_config_dir.config_dir_version); + assert.strictEqual(actual_config_dir_health.upgrade_package_version, expected_config_dir.upgrade_package_version); + const expected_upgrade_status = expected_config_dir.in_progress_upgrade || expected_config_dir.upgrade_history?.last_failure; + assert_upgrade_status(actual_config_dir_health.upgrade_status, expected_upgrade_status); +} + +/** + * assert_upgrade_status asserts there the actual and expected upgrade_status + * @param {Object} actual_upgrade_status + * @param {Object} [expected_upgrade_status] + */ +function assert_upgrade_status(actual_upgrade_status, expected_upgrade_status = default_mock_upgrade_status.message) { + const actual_upgrade_status_res = actual_upgrade_status.in_progress_upgrade || + actual_upgrade_status.last_failure || + actual_upgrade_status.message; + assert.deepStrictEqual(actual_upgrade_status_res, expected_upgrade_status); +} + +/** + * restore_health_if_needed restores health obj functions if needed + * @param {*} health_obj + */ +function restore_health_if_needed(health_obj) { + if (health_obj?.get_service_state?.restore) health_obj.get_service_state.restore(); + if (health_obj?.get_endpoint_response?.restore) health_obj.get_endpoint_response.restore(); + if (health_obj?.config_fs?.get_system_config_file?.restore) health_obj.config_fs.get_system_config_file.restore(); + if (health_obj?.get_service_memory_usage?.restore) health_obj.get_service_memory_usage.restore(); +} + +/** + * set_mock_functions sets mock functions used by the health script + * the second param is an object having the name of the mock functions as the keys and + * the value is an array of responses by the order of their call + * @param {Object} Health + * @param {{get_endpoint_response?: Object[], get_service_state?: Object[], + * get_system_config_file?: Object[], get_service_memory_usage?: Object[]}} mock_function_responses + */ +function set_mock_functions(Health, mock_function_responses) { + for (const mock_function_name of Object.keys(mock_function_responses)) { + const mock_function_responses_arr = mock_function_responses[mock_function_name]; + const obj_to_stub = mock_function_name === 'get_system_config_file' ? Health.config_fs : Health; + + if (obj_to_stub[mock_function_name]?.restore) obj_to_stub[mock_function_name]?.restore(); + const stub = sinon.stub(obj_to_stub, mock_function_name); + for (let i = 0; i < mock_function_responses_arr.length; i++) { + stub.onCall(i).returns(Promise.resolve(mock_function_responses_arr[i])); + } + } +} diff --git a/src/upgrade/nc_upgrade_manager.js b/src/upgrade/nc_upgrade_manager.js index 83bde5fa57..e0630fe94b 100644 --- a/src/upgrade/nc_upgrade_manager.js +++ b/src/upgrade/nc_upgrade_manager.js @@ -6,10 +6,9 @@ const _ = require('lodash'); const path = require('path'); const util = require('util'); const pkg = require('../../package.json'); -const dbg = require('../util/debug_module')('UPGRADE'); +const dbg = require('../util/debug_module')(__filename); const { should_upgrade, run_upgrade_scripts, version_compare } = require('./upgrade_utils'); -dbg.set_process_name('Upgrade'); const hostname = os.hostname(); const CONFIG_DIR_LOCKED = 'CONFIG_DIR_LOCKED'; diff --git a/src/upgrade/upgrade_utils.js b/src/upgrade/upgrade_utils.js index 453548eb1c..ff34af4d15 100644 --- a/src/upgrade/upgrade_utils.js +++ b/src/upgrade/upgrade_utils.js @@ -4,9 +4,7 @@ const fs = require('fs'); const _ = require('lodash'); const path = require('path'); -const dbg = require('../util/debug_module')('UPGRADE'); -dbg.set_process_name('Upgrade'); - +const dbg = require('../util/debug_module')(__filename); /** * @param {string} ver From 6afae5d6ec7481646bbf0d78723fe110ab4ed3a8 Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:28:06 +0200 Subject: [PATCH 6/6] NSFS | Replace ChunkFS with FileWriter Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> (cherry picked from commit a451243179968472dca31fc3322b1e311cf3d64c) --- config.js | 2 + src/sdk/namespace_fs.js | 35 ++--- src/test/unit_tests/index.js | 2 +- src/test/unit_tests/nc_index.js | 2 +- src/test/unit_tests/test_chunk_fs.js | 34 ----- src/test/unit_tests/test_file_writer.js | 69 +++++++++ ...k_fs_hashing.js => file_writer_hashing.js} | 51 +++---- src/util/chunk_fs.js | 111 -------------- src/util/file_writer.js | 140 ++++++++++++++++++ 9 files changed, 252 insertions(+), 194 deletions(-) delete mode 100644 src/test/unit_tests/test_chunk_fs.js create mode 100644 src/test/unit_tests/test_file_writer.js rename src/tools/{chunk_fs_hashing.js => file_writer_hashing.js} (76%) delete mode 100644 src/util/chunk_fs.js create mode 100644 src/util/file_writer.js diff --git a/config.js b/config.js index b7429fe55f..1dd31806ff 100644 --- a/config.js +++ b/config.js @@ -882,6 +882,8 @@ config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10; // anonymous account name config.ANONYMOUS_ACCOUNT_NAME = 'anonymous'; +config.NFSF_UPLOAD_STREAM_MEM_THRESHOLD = 8 * 1024 * 1024; + //////////////////////////// // NSFS NON CONTAINERIZED // //////////////////////////// diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index 53e12965f7..fabe042ed3 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -19,7 +19,7 @@ const stream_utils = require('../util/stream_utils'); const buffer_utils = require('../util/buffer_utils'); const size_utils = require('../util/size_utils'); const native_fs_utils = require('../util/native_fs_utils'); -const ChunkFS = require('../util/chunk_fs'); +const FileWriter = require('../util/file_writer'); const LRUCache = require('../util/lru_cache'); const nb_native = require('../util/nb_native'); const RpcError = require('../rpc/rpc_error'); @@ -1564,36 +1564,33 @@ class NamespaceFS { // Can be finetuned further on if needed and inserting the Semaphore logic inside // Instead of wrapping the whole _upload_stream function (q_buffers lives outside of the data scope of the stream) async _upload_stream({ fs_context, params, target_file, object_sdk, offset }) { - const { source_stream, copy_source } = params; + const { copy_source } = params; try { // Not using async iterators with ReadableStreams due to unsettled promises issues on abort/destroy const md5_enabled = this._is_force_md5_enabled(object_sdk); - const chunk_fs = new ChunkFS({ + const file_writer = new FileWriter({ target_file, fs_context, - stats: this.stats, - namespace_resource_id: this.namespace_resource_id, - md5_enabled, offset, + md5_enabled, + stats: this.stats, bucket: params.bucket, - large_buf_size: multi_buffer_pool.get_buffers_pool(undefined).buf_size + large_buf_size: multi_buffer_pool.get_buffers_pool(undefined).buf_size, + namespace_resource_id: this.namespace_resource_id, }); - chunk_fs.on('error', err1 => dbg.error('namespace_fs._upload_stream: error occured on stream ChunkFS: ', err1)); - chunk_fs.on('finish', arg => dbg.error('namespace_fs._upload_stream: finish occured on stream ChunkFS: ', arg)); - chunk_fs.on('close', arg => dbg.error('namespace_fs._upload_stream: close occured on stream ChunkFS: ', arg)); + file_writer.on('error', err => dbg.error('namespace_fs._upload_stream: error occured on FileWriter: ', err)); + file_writer.on('finish', arg => dbg.log1('namespace_fs._upload_stream: finish occured on stream FileWriter: ', arg)); + file_writer.on('close', arg => dbg.log1('namespace_fs._upload_stream: close occured on stream FileWriter: ', arg)); + if (copy_source) { - // ChunkFS is a Transform stream, however read_object_stream expects a write stream. call resume to close the read part - // we need to close both read and write parts for Transform stream to properly close and release resorces - chunk_fs.resume(); - await this.read_object_stream(copy_source, object_sdk, chunk_fs); + await this.read_object_stream(copy_source, object_sdk, file_writer); } else if (params.source_params) { - chunk_fs.resume(); - await params.source_ns.read_object_stream(params.source_params, object_sdk, chunk_fs); + await params.source_ns.read_object_stream(params.source_params, object_sdk, file_writer); } else { - await stream_utils.pipeline([source_stream, chunk_fs]); - await stream_utils.wait_finished(chunk_fs); + await stream_utils.pipeline([params.source_stream, file_writer]); + await stream_utils.wait_finished(file_writer); } - return { digest: chunk_fs.digest, total_bytes: chunk_fs.total_bytes }; + return { digest: file_writer.digest, total_bytes: file_writer.total_bytes }; } catch (error) { dbg.error('_upload_stream had error: ', error); throw error; diff --git a/src/test/unit_tests/index.js b/src/test/unit_tests/index.js index ac34e80579..22ecd77f58 100644 --- a/src/test/unit_tests/index.js +++ b/src/test/unit_tests/index.js @@ -57,7 +57,7 @@ require('./test_bucket_chunks_builder'); require('./test_mirror_writer'); require('./test_namespace_fs'); require('./test_ns_list_objects'); -require('./test_chunk_fs'); +require('./test_file_writer'); require('./test_namespace_fs_mpu'); require('./test_nb_native_fs'); require('./test_s3select'); diff --git a/src/test/unit_tests/nc_index.js b/src/test/unit_tests/nc_index.js index 49cc875263..d8d38e5241 100644 --- a/src/test/unit_tests/nc_index.js +++ b/src/test/unit_tests/nc_index.js @@ -7,7 +7,7 @@ coretest.setup(); require('./test_namespace_fs'); require('./test_ns_list_objects'); -require('./test_chunk_fs'); +require('./test_file_writer'); require('./test_namespace_fs_mpu'); require('./test_nb_native_fs'); require('./test_nc_nsfs_cli'); diff --git a/src/test/unit_tests/test_chunk_fs.js b/src/test/unit_tests/test_chunk_fs.js deleted file mode 100644 index 3885e21ed7..0000000000 --- a/src/test/unit_tests/test_chunk_fs.js +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright (C) 2020 NooBaa */ -/* eslint-disable no-invalid-this */ -'use strict'; - -const mocha = require('mocha'); -const chunk_fs_hashing = require('../../tools/chunk_fs_hashing'); - -mocha.describe('ChunkFS', function() { - const RUN_TIMEOUT = 10 * 60 * 1000; - - mocha.it('Concurrent ChunkFS with hash target', async function() { - const self = this; - self.timeout(RUN_TIMEOUT); - await chunk_fs_hashing.hash_target(); - }); - - mocha.it('Concurrent ChunkFS with file target', async function() { - const self = this; - self.timeout(RUN_TIMEOUT); - await chunk_fs_hashing.file_target(); - }); - - mocha.it('Concurrent ChunkFS with file target - produce num_chunks > 1024 && total_chunks_size < config.NSFS_BUF_SIZE_L', async function() { - const self = this; - self.timeout(RUN_TIMEOUT); - // The goal of this test is to produce num_chunks > 1024 && total_chunks_size < config.NSFS_BUF_SIZE_L - // so we will flush buffers because of reaching max num of buffers and not because we reached the max NSFS buf size - // chunk size = 100, num_chunks = (10 * 1024 * 1024)/100 < 104587, 104587 = num_chunks > 1024 - // chunk size = 100, total_chunks_size after having 1024 chunks is = 100 * 1024 < config.NSFS_BUF_SIZE_L - const chunk_size = 100; - const parts_s = 50; - await chunk_fs_hashing.file_target(chunk_size, parts_s); - }); -}); diff --git a/src/test/unit_tests/test_file_writer.js b/src/test/unit_tests/test_file_writer.js new file mode 100644 index 0000000000..c3773de62b --- /dev/null +++ b/src/test/unit_tests/test_file_writer.js @@ -0,0 +1,69 @@ +/* Copyright (C) 2020 NooBaa */ +/* eslint-disable no-invalid-this */ +'use strict'; + +const mocha = require('mocha'); +const config = require('../../../config'); +const file_writer_hashing = require('../../tools/file_writer_hashing'); +const orig_iov_max = config.NSFS_DEFAULT_IOV_MAX; + +// on iov_max small tests we need to use smaller amount of parts and chunks to ensure that the test will finish +// in a reasonable period of time because we will flush max 1/2 buffers at a time. +const small_iov_num_parts = 20; + + +mocha.describe('FileWriter', function() { + const RUN_TIMEOUT = 10 * 60 * 1000; + + mocha.afterEach(function() { + config.NSFS_DEFAULT_IOV_MAX = orig_iov_max; + }); + + mocha.it('Concurrent FileWriter with hash target', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.hash_target(); + }); + + mocha.it('Concurrent FileWriter with file target', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.file_target(); + }); + + mocha.it('Concurrent FileWriter with hash target - iov_max=1', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.hash_target(undefined, small_iov_num_parts, 1); + }); + + mocha.it('Concurrent FileWriter with file target - iov_max=1', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.file_target(undefined, small_iov_num_parts, 1); + }); + + mocha.it('Concurrent FileWriter with hash target - iov_max=2', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.hash_target(undefined, small_iov_num_parts, 2); + }); + + mocha.it('Concurrent FileWriter with file target - iov_max=2', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + await file_writer_hashing.file_target(undefined, small_iov_num_parts, 2); + }); + + mocha.it('Concurrent FileWriter with file target - produce num_chunks > 1024 && total_chunks_size < config.NSFS_BUF_SIZE_L', async function() { + const self = this; + self.timeout(RUN_TIMEOUT); + // The goal of this test is to produce num_chunks > 1024 && total_chunks_size < config.NSFS_BUF_SIZE_L + // so we will flush buffers because of reaching max num of buffers and not because we reached the max NSFS buf size + // chunk size = 100, num_chunks = (10 * 1024 * 1024)/100 < 104587, 104587 = num_chunks > 1024 + // chunk size = 100, total_chunks_size after having 1024 chunks is = 100 * 1024 < config.NSFS_BUF_SIZE_L + const chunk_size = 100; + const parts_s = 50; + await file_writer_hashing.file_target(chunk_size, parts_s); + }); +}); diff --git a/src/tools/chunk_fs_hashing.js b/src/tools/file_writer_hashing.js similarity index 76% rename from src/tools/chunk_fs_hashing.js rename to src/tools/file_writer_hashing.js index 09a3465720..e3f2c980dc 100644 --- a/src/tools/chunk_fs_hashing.js +++ b/src/tools/file_writer_hashing.js @@ -3,7 +3,7 @@ const crypto = require('crypto'); const assert = require('assert'); -const ChunkFS = require('../util/chunk_fs'); +const FileWriter = require('../util/file_writer'); const config = require('../../config'); const nb_native = require('../util/nb_native'); const stream_utils = require('../util/stream_utils'); @@ -19,7 +19,8 @@ const PARTS = Number(argv.parts) || 1000; const CONCURRENCY = Number(argv.concurrency) || 20; const CHUNK = Number(argv.chunk) || 16 * 1024; const PART_SIZE = Number(argv.part_size) || 20 * 1024 * 1024; -const F_PREFIX = argv.dst_folder || '/tmp/chunk_fs_hashing/'; +const F_PREFIX = argv.dst_folder || '/tmp/file_writer_hashing/'; +const IOV_MAX = argv.iov_max || config.NSFS_DEFAULT_IOV_MAX; const DEFAULT_FS_CONFIG = { uid: Number(argv.uid) || process.getuid(), @@ -28,12 +29,6 @@ const DEFAULT_FS_CONFIG = { warn_threshold_ms: 100, }; -const DUMMY_RPC = { - object: { - update_endpoint_stats: (...params) => null - } -}; - const XATTR_USER_PREFIX = 'user.'; // TODO: In order to verify validity add content_md5_mtime as well const XATTR_MD5_KEY = XATTR_USER_PREFIX + 'content_md5'; @@ -64,41 +59,42 @@ function assign_md5_to_fs_xattr(md5_digest, fs_xattr) { return fs_xattr; } -async function hash_target() { - await P.map_with_concurrency(CONCURRENCY, Array(PARTS).fill(), async () => { +async function hash_target(chunk_size = CHUNK, parts = PARTS, iov_max = IOV_MAX) { + config.NSFS_DEFAULT_IOV_MAX = iov_max; + await P.map_with_concurrency(CONCURRENCY, Array(parts).fill(), async () => { const data = crypto.randomBytes(PART_SIZE); const content_md5 = crypto.createHash('md5').update(data).digest('hex'); // Using async generator function in order to push data in small chunks const source_stream = stream.Readable.from(async function*() { - for (let i = 0; i < data.length; i += CHUNK) { - yield data.slice(i, i + CHUNK); + for (let i = 0; i < data.length; i += chunk_size) { + yield data.slice(i, i + chunk_size); } }()); const target = new TargetHash(); - const chunk_fs = new ChunkFS({ + const file_writer = new FileWriter({ target_file: target, fs_context: DEFAULT_FS_CONFIG, - rpc_client: DUMMY_RPC, namespace_resource_id: 'MajesticSloth' }); - await stream_utils.pipeline([source_stream, chunk_fs]); - await stream_utils.wait_finished(chunk_fs); + await stream_utils.pipeline([source_stream, file_writer]); + await stream_utils.wait_finished(file_writer); const write_hash = target.digest(); console.log( 'Hash target', - `NativeMD5=${chunk_fs.digest}`, + `NativeMD5=${file_writer.digest}`, `DataWriteCryptoMD5=${write_hash}`, `DataOriginMD5=${content_md5}`, ); assert.strictEqual(content_md5, write_hash); if (config.NSFS_CALCULATE_MD5) { - assert.strictEqual(chunk_fs.digest, content_md5); - assert.strictEqual(chunk_fs.digest, write_hash); + assert.strictEqual(file_writer.digest, content_md5); + assert.strictEqual(file_writer.digest, write_hash); } }); } -async function file_target(chunk_size = CHUNK, parts = PARTS) { +async function file_target(chunk_size = CHUNK, parts = PARTS, iov_max = IOV_MAX) { + config.NSFS_DEFAULT_IOV_MAX = iov_max; fs.mkdirSync(F_PREFIX); await P.map_with_concurrency(CONCURRENCY, Array(parts).fill(), async () => { let target_file; @@ -113,32 +109,31 @@ async function file_target(chunk_size = CHUNK, parts = PARTS) { yield data.slice(i, i + chunk_size); } }()); - const chunk_fs = new ChunkFS({ + const file_writer = new FileWriter({ target_file, fs_context: DEFAULT_FS_CONFIG, - rpc_client: DUMMY_RPC, namespace_resource_id: 'MajesticSloth' }); - await stream_utils.pipeline([source_stream, chunk_fs]); - await stream_utils.wait_finished(chunk_fs); + await stream_utils.pipeline([source_stream, file_writer]); + await stream_utils.wait_finished(file_writer); if (XATTR) { await target_file.replacexattr( DEFAULT_FS_CONFIG, - assign_md5_to_fs_xattr(chunk_fs.digest, {}) + assign_md5_to_fs_xattr(file_writer.digest, {}) ); } if (FSYNC) await target_file.fsync(DEFAULT_FS_CONFIG); const write_hash = crypto.createHash('md5').update(fs.readFileSync(F_TARGET)).digest('hex'); console.log( 'File target', - `NativeMD5=${chunk_fs.digest}`, + `NativeMD5=${file_writer.digest}`, `DataWriteMD5=${write_hash}`, `DataOriginMD5=${content_md5}`, ); assert.strictEqual(content_md5, write_hash); if (config.NSFS_CALCULATE_MD5) { - assert.strictEqual(chunk_fs.digest, content_md5); - assert.strictEqual(chunk_fs.digest, write_hash); + assert.strictEqual(file_writer.digest, content_md5); + assert.strictEqual(file_writer.digest, write_hash); } // Leave parts on error fs.rmSync(F_TARGET); diff --git a/src/util/chunk_fs.js b/src/util/chunk_fs.js deleted file mode 100644 index e60c9aa9e0..0000000000 --- a/src/util/chunk_fs.js +++ /dev/null @@ -1,111 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const stream = require('stream'); -const config = require('../../config'); -const nb_native = require('./nb_native'); -const dbg = require('../util/debug_module')(__filename); - -/** - * - * ChunkFS - * - * Calculates etag and writes stream data to the filesystem batching data buffers - * - */ -class ChunkFS extends stream.Transform { - - /** - * @param {{ - * target_file: object, - * fs_context: object, - * namespace_resource_id: string, - * md5_enabled: boolean, - * stats: import('../sdk/endpoint_stats_collector').EndpointStatsCollector, - * offset?: number, - * bucket?: string, - * large_buf_size?: number, - * }} params - */ - constructor({ target_file, fs_context, namespace_resource_id, md5_enabled, stats, offset, bucket, large_buf_size }) { - super(); - this.q_buffers = []; - this.q_size = 0; - this.MD5Async = md5_enabled ? new (nb_native().crypto.MD5Async)() : undefined; - this.target_file = target_file; - this.fs_context = fs_context; - this.count = 1; - this.total_bytes = 0; - this.offset = offset; - this.namespace_resource_id = namespace_resource_id; - this.stats = stats; - this._total_num_buffers = 0; - const platform_iov_max = nb_native().fs.PLATFORM_IOV_MAX; - this.iov_max = platform_iov_max ? Math.min(platform_iov_max, config.NSFS_DEFAULT_IOV_MAX) : config.NSFS_DEFAULT_IOV_MAX; - this.bucket = bucket; - this.large_buf_size = large_buf_size || config.NSFS_BUF_SIZE_L; - } - - async _transform(chunk, encoding, callback) { - try { - if (this.MD5Async) await this.MD5Async.update(chunk); - this.stats?.update_nsfs_write_stats({ - namespace_resource_id: this.namespace_resource_id, - size: chunk.length, - count: this.count, - bucket_name: this.bucket, - }); - this.count = 0; - while (chunk && chunk.length) { - const available_size = this.large_buf_size - this.q_size; - const buf = (available_size < chunk.length) ? chunk.slice(0, available_size) : chunk; - this.q_buffers.push(buf); - this.q_size += buf.length; - // Should flush when num of chunks equals to max iov which is the limit according to https://linux.die.net/man/2/writev - // or when q_size equals to config.NSFS_BUF_SIZE_L, but added greater than just in case - if (this.q_buffers.length === this.iov_max || this.q_size >= config.NSFS_BUF_SIZE_L) await this._flush_buffers(); - chunk = (available_size < chunk.length) ? chunk.slice(available_size) : null; - } - return callback(); - } catch (error) { - console.error('ChunkFS _transform failed', this.q_size, this._total_num_buffers, error); - return callback(error); - } - } - - async _flush(callback) { - // wait before the last writev to finish - await this._flush_buffers(callback); - } - - // callback will be passed only at the end of the stream by _flush() - // while this function is called without callback during _transform() and returns a promise. - async _flush_buffers(callback) { - try { - if (this.q_buffers.length) { - const buffers_to_write = this.q_buffers; - const size_to_write = this.q_size; - this.q_buffers = []; - this.q_size = 0; - dbg.log1(`Chunk_fs._flush_buffers: writing ${buffers_to_write.length} buffers, total size is ${size_to_write}`); - await this.target_file.writev(this.fs_context, buffers_to_write, this.offset); - // Hold the ref on the buffers from the JS side - this._total_num_buffers += buffers_to_write.length; - this.total_bytes += size_to_write; - if (this.offset >= 0) this.offset += size_to_write; - } - if (callback) { - if (this.MD5Async) this.digest = (await this.MD5Async.digest()).toString('hex'); - return callback(); - } - } catch (error) { - console.error('ChunkFS _flush_buffers failed', this.q_size, this._total_num_buffers, error); - if (callback) { - return callback(error); - } - throw error; - } - } -} - -module.exports = ChunkFS; diff --git a/src/util/file_writer.js b/src/util/file_writer.js new file mode 100644 index 0000000000..c8df126719 --- /dev/null +++ b/src/util/file_writer.js @@ -0,0 +1,140 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + +const stream = require('stream'); +const config = require('../../config'); +const nb_native = require('./nb_native'); +const dbg = require('../util/debug_module')(__filename); + +/** + * FileWriter is a Writable stream that write data to a filesystem file, + * with optional calculation of md5 for etag. + */ +class FileWriter extends stream.Writable { + + /** + * @param {{ + * target_file: object, + * fs_context: object, + * namespace_resource_id: string, + * md5_enabled: boolean, + * stats: import('../sdk/endpoint_stats_collector').EndpointStatsCollector, + * offset?: number, + * bucket?: string, + * large_buf_size?: number, + * }} params + */ + constructor({ target_file, fs_context, namespace_resource_id, md5_enabled, stats, offset, bucket, large_buf_size }) { + super({ highWaterMark: config.NFSF_UPLOAD_STREAM_MEM_THRESHOLD }); + this.target_file = target_file; + this.fs_context = fs_context; + this.offset = offset; + this.total_bytes = 0; + this.count_once = 1; + this.stats = stats; + this.bucket = bucket; + this.namespace_resource_id = namespace_resource_id; + this.large_buf_size = large_buf_size || config.NSFS_BUF_SIZE_L; + this.MD5Async = md5_enabled ? new (nb_native().crypto.MD5Async)() : undefined; + const platform_iov_max = nb_native().fs.PLATFORM_IOV_MAX; + this.iov_max = platform_iov_max ? Math.min(platform_iov_max, config.NSFS_DEFAULT_IOV_MAX) : config.NSFS_DEFAULT_IOV_MAX; + } + + /** + * @param {number} size + */ + _update_stats(size) { + const count = this.count_once; + this.count_once = 0; // counting the entire operation just once + this.stats?.update_nsfs_write_stats({ + namespace_resource_id: this.namespace_resource_id, + size, + count, + bucket_name: this.bucket, + }); + } + + /** + * @param {Buffer[]} buffers + * @param {number} size + */ + async _update_md5(buffers, size) { + // TODO optimize by calling once with all buffers + for (const buf of buffers) { + await this.MD5Async.update(buf); + } + } + + /** + * @param {Buffer[]} buffers + * @param {number} size + */ + async _write_all_buffers(buffers, size) { + if (buffers.length <= this.iov_max) { + await this._write_to_file(buffers, size); + } else { + let iov_start = 0; + while (iov_start < buffers.length) { + const iov_end = Math.min(buffers.length, iov_start + this.iov_max); + const buffers_to_write = buffers.slice(iov_start, iov_end); + const size_to_write = buffers_to_write.reduce((s, b) => s + b.length, 0); + await this._write_to_file(buffers_to_write, size_to_write); + iov_start = iov_end; + } + } + } + + /** + * @param {Buffer[]} buffers + * @param {number} size + */ + async _write_to_file(buffers, size) { + dbg.log1(`FileWriter._write_to_file: buffers ${buffers.length} size ${size} offset ${this.offset}`); + await this.target_file.writev(this.fs_context, buffers, this.offset); + if (this.offset >= 0) this.offset += size; // when offset<0 we just append + this.total_bytes += size; + } + + /** + * @param {Array<{ chunk: Buffer; encoding: BufferEncoding; }>} chunks + * @param {(error?: Error | null) => void} callback + */ + async _writev(chunks, callback) { + try { + let size = 0; + const buffers = chunks.map(it => { + size += it.chunk.length; + return it.chunk; + }); + await Promise.all([ + this.MD5Async && this._update_md5(buffers, size), + this._write_all_buffers(buffers, size), + ]); + this._update_stats(size); + return callback(); + } catch (err) { + console.error('FileWriter._writev: failed', err); + return callback(err); + } + } + + /** + * @param {(error?: Error | null) => void} callback + */ + async _final(callback) { + try { + if (this.MD5Async) { + const digest = await this.MD5Async.digest(); + this.digest = digest.toString('hex'); + } + + return callback(); + } catch (err) { + console.error('FileWriter._final: failed', err); + return callback(err); + } + } + +} + +module.exports = FileWriter;