From 027be1b157ec3fa57364fa2e4880d934988539a9 Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:21:34 +0200 Subject: [PATCH] nc prepare services info and analyze fork Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> --- src/manage_nsfs/diagnose.js | 7 +- src/manage_nsfs/health.js | 87 +++++++++++++++--------- src/tools/diagnostics/analyze_network.js | 58 +++++++++++++--- 3 files changed, 107 insertions(+), 45 deletions(-) diff --git a/src/manage_nsfs/diagnose.js b/src/manage_nsfs/diagnose.js index 6a6a561a9b..7d34b2e49e 100644 --- a/src/manage_nsfs/diagnose.js +++ b/src/manage_nsfs/diagnose.js @@ -10,6 +10,7 @@ const { DIAGNOSE_ACTIONS } = require('./manage_nsfs_constants'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; const { throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils'); const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; +const analyze_network = require('../tools/diagnostics/analyze_network'); /** * manage_diagnose_operations handles cli diagnose operations @@ -29,9 +30,9 @@ async function manage_diagnose_operations(action, user_input, config_fs) { case DIAGNOSE_ACTIONS.METRICS: await gather_metrics(); break; - // case DIAGNOSE_ACTIONS.NETWORK: - // await test_network(); - // break; + case DIAGNOSE_ACTIONS.NETWORK: + await analyze_network.get_network_status(user_input, config_fs); + break; default: throw_cli_error(ManageCLIError.InvalidAction); } diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 65ccf24da3..c601d8781d 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -235,28 +235,13 @@ class NSFSHealth { return service_health; } - async make_endpoint_health_request(url_path) { - const response = await make_https_request({ - HOSTNAME, - port: this.https_port, - path: url_path, - method: 'GET', - rejectUnauthorized: false, - }); - if (response && response.statusCode === 200) { - const buffer = await read_stream_join(response); - const body = buffer.toString('utf8'); - return JSON.parse(body); - } - } - async get_endpoint_fork_response() { - let url_path = '/total_fork_count'; - const worker_ids = []; + const url_path = '/total_fork_count'; + let worker_ids = []; let total_fork_count = 0; let response; try { - const fork_count_response = await this.make_endpoint_health_request(url_path); + const fork_count_response = await make_endpoint_health_request(url_path, this.https_port); if (!fork_count_response) { return { response: fork_response_code.NOT_RUNNING, @@ -266,20 +251,7 @@ class NSFSHealth { } total_fork_count = fork_count_response.fork_count; if (total_fork_count > 0) { - url_path = '/endpoint_fork_id'; - await P.retry({ - attempts: total_fork_count * 2, - delay_ms: 1, - func: async () => { - const fork_id_response = await this.make_endpoint_health_request(url_path); - if (fork_id_response.worker_id && !worker_ids.includes(fork_id_response.worker_id)) { - worker_ids.push(fork_id_response.worker_id); - } - if (worker_ids.length < total_fork_count) { - throw new Error('Number of running forks is less than the expected fork count.'); - } - } - }); + worker_ids = await call_forks(total_fork_count, this.https_port); if (worker_ids.length === total_fork_count) { response = fork_response_code.RUNNING; } else { @@ -679,5 +651,56 @@ function _should_skip_health_access_check() { return config.NC_DISABLE_HEALTH_ACCESS_CHECK || config.NC_DISABLE_ACCESS_CHECK; } +/** + * make_endpoint_health_request runs https request to the endpoint server + * @param {String} url_path + * @param {Number} https_port + * @returns {Promise} + * // TODO: need to return on error + */ +async function make_endpoint_health_request(url_path, https_port) { + const response = await make_https_request({ + HOSTNAME, + port: https_port, + path: url_path, + method: 'GET', + rejectUnauthorized: false, + }); + if (response && response.statusCode === 200) { + const buffer = await read_stream_join(response); + const body = buffer.toString('utf8'); + return JSON.parse(body); + } +} + +/** + * call_forks executes http request to the endpoint and adds the worker_id to an array + * throws an error as long as the worker ids count is smaller than fork_count + * TODO: consider removing the forks check from here and have it on the network analyzer/both + * @param {Number} fork_count + * @param {Number} https_port + * @returns {Promise} + */ +async function call_forks(fork_count, https_port) { + if (fork_count > 0) { + const worker_ids = []; + await P.retry({ + attempts: fork_count * 2, + delay_ms: 1, + func: async () => { + const fork_id_response = await make_endpoint_health_request('/endpoint_fork_id', https_port); + if (fork_id_response.worker_id && !worker_ids.includes(fork_id_response.worker_id)) { + worker_ids.push(fork_id_response.worker_id); + } + if (worker_ids.length < fork_count) { + throw new Error('Number of running forks is less than the expected fork count.'); + } + } + }); + return worker_ids; + } +} + +exports.call_forks = call_forks; exports.get_health_status = get_health_status; exports.NSFSHealth = NSFSHealth; diff --git a/src/tools/diagnostics/analyze_network.js b/src/tools/diagnostics/analyze_network.js index c7246f9186..c9c3ca1d47 100644 --- a/src/tools/diagnostics/analyze_network.js +++ b/src/tools/diagnostics/analyze_network.js @@ -3,16 +3,28 @@ const dbg = require('../../util/debug_module')(__filename); dbg.set_process_name('analyze_network'); +const { is_hostname } = require('../../util/net_utils'); +const config = require('../../../config'); +const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); +const { ManageCLIResponse } = require('../../manage_nsfs/manage_nsfs_cli_responses'); +const { throw_cli_error, write_stdout_response } = require('../../manage_nsfs/manage_nsfs_cli_utils'); -async function main(argv) { +async function get_network_status(argv, config_fs) { + const deployment_type = argv.deployment_type; + const is_nc_deployment = deployment_type === 'nc'; try { dbg.log0('starting to analyze network'); - const deployment_type = argv.deployment_type; - if (deployment_type === 'nc') await test_nc_network(); - else await test_network(); + if (is_nc_deployment) { + const nc_network_status = await test_nc_network(config_fs); + write_stdout_response(ManageCLIResponse.HealthStatus, nc_network_status); + } else { + const network_status = await test_network(); + console.log('network_status', network_status); + } } catch (err) { - process.exit(1); + dbg.error('Health: exit on error', err.stack || err); + if (is_nc_deployment) throw_cli_error({ ...ManageCLIError.HealthStatusFailed, cause: err }); } process.exit(0); } @@ -29,9 +41,9 @@ const ANALYZE_FUNCTION_BY_SERVICE_TYPE = { /** * */ -async function test_nc_network() { +async function test_nc_network(config_fs) { await analyze_forks(); - const services_info = await nc_prepare_service_info(); + const services_info = await nc_prepare_services_info(config_fs); const nc_enabled_services = ['S3', 'METRICS']; // IAM, STS for (const service of nc_enabled_services) { await analyze_service(service, services_info[service]); @@ -107,14 +119,40 @@ async function analyze_metrics(service_info) { ///////////////////////////////// async function analyze_forks() { + try { + const url_path = '/total_fork_count'; + const fork_count_response = await make_endpoint_health_request(url_path, this.https_port); + const num_of_forks = fork_count_response.fork_count; -} + const responsive_forks = await call_forks(num_of_forks, ); + } catch (err) { -async function nc_prepare_service_info() { + } + return { responsive_forks: } +} +/** + * nc_prepare_services_info creates services info object + * @param {import('../../sdk/config_fs').ConfigFS} config_fs + * @returns {Promise} + */ +async function nc_prepare_services_info(config_fs) { + const system_data = await config_fs.get_system_config_file({ silent_if_missing: true }); + const nc_services_info = []; + for (const hostname of Object.keys(system_data)) { + if (!is_hostname(hostname)) continue; + // TODO: + // 1. need to take the port from config.json per host and not from the general config + // 2. add other service in the future + const s3_ssl_info = { service: 's3', hostname, port: config.ENDPOINT_PORT, secure: false }; + const s3_info = { service: 's3', hostname, port: config.ENDPOINT_SSL_PORT, secure: true }; + const metrics_info = { service: 'metrics', hostname, port: config.EP_METRICS_SERVER_PORT, secure: false }; + nc_services_info.push([s3_ssl_info, s3_info, metrics_info]); + } + return nc_services_info; } if (require.main === module) { - main(); + get_network_status(); }