diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 25629b0273..12a83c4112 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -14,6 +14,7 @@ const { TYPES } = require('./manage_nsfs_constants'); const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils'); const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; +const { CONFIG_DIR_LOCKED } = require('../upgrade/nc_upgrade_manager'); const HOSTNAME = 'localhost'; @@ -125,6 +126,7 @@ class NSFSHealth { const error_code = await this.get_error_code(service_status, pid, response_code); if (this.all_bucket_details) bucket_details = await this.get_bucket_status(); if (this.all_account_details) account_details = await this.get_account_status(); + const system_data = await this.config_fs.get_system_config_file({ silent_if_missing: true }); const health = { service_name: NOOBAA_SERVICE, status: service_health, @@ -136,6 +138,8 @@ class NSFSHealth { endpoint_state, error_type: health_errors_tyes.TEMPORARY, }, + config_directory_status: this._get_config_dir_status(system_data), + upgrade_status: this._get_upgrade_status(system_data), accounts_status: { invalid_accounts: account_details === undefined ? undefined : account_details.invalid_storages, valid_accounts: account_details === undefined ? undefined : account_details.valid_storages, @@ -145,7 +149,7 @@ class NSFSHealth { invalid_buckets: bucket_details === undefined ? undefined : bucket_details.invalid_storages, valid_buckets: bucket_details === undefined ? undefined : bucket_details.valid_storages, error_type: health_errors_tyes.PERSISTENT, - } + }, } }; if (!this.all_account_details) delete health.checks.accounts_status; @@ -421,6 +425,38 @@ class NSFSHealth { err_obj }; } + + /** + * _get_config_dir_status returns the config directory phase, version and equivalent package_version + * @param {Object} system_data + * @returns {Object} + */ + _get_config_dir_status(system_data) { + if (!system_data) return { error: 'system data is missing' }; + const config_dir_data = system_data.config_directory; + if (!config_dir_data) return { error: 'config directory data is missing' }; + return { + phase: config_dir_data.phase, + config_dir_version: config_dir_data.config_dir_version, + upgrade_package_version: config_dir_data.upgrade_package_version, + in_progress_upgrade: config_dir_data.in_progress_upgrade + }; + } + + /** + * get_upgrade_status returns the status of an ongoing upgrade, if valid it returns an object with upgrade details + * if upgrade is not ongoing but config dir is locked in one of the hosts, the error details will return + * @param {Object} system_data + * @returns {Object} + */ + _get_upgrade_status(system_data) { + if (!system_data) return { error: 'system data is missing' }; + const config_dir_data = system_data.config_directory; + if (!config_dir_data) return { error: 'config directory data is missing' }; + if (!config_dir_data.in_progress_upgrade && config_dir_data.phase === CONFIG_DIR_LOCKED) { + return config_dir_data.upgrade_history.last_failure; + } + } } async function get_health_status(argv, config_fs) { diff --git a/src/test/unit_tests/nc_index.js b/src/test/unit_tests/nc_index.js index 145120e3b8..c6dd4a3cef 100644 --- a/src/test/unit_tests/nc_index.js +++ b/src/test/unit_tests/nc_index.js @@ -11,7 +11,7 @@ require('./test_chunk_fs'); require('./test_namespace_fs_mpu'); require('./test_nb_native_fs'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); require('./test_nsfs_access'); require('./test_bucketspace'); require('./test_bucketspace_fs'); diff --git a/src/test/unit_tests/sudo_index.js b/src/test/unit_tests/sudo_index.js index 77b10c6120..8d5f109a2a 100644 --- a/src/test/unit_tests/sudo_index.js +++ b/src/test/unit_tests/sudo_index.js @@ -21,5 +21,5 @@ require('./test_bucketspace_versioning'); require('./test_bucketspace_fs'); require('./test_nsfs_versioning'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_health.js similarity index 81% rename from src/test/unit_tests/test_nc_nsfs_health.js rename to src/test/unit_tests/test_nc_health.js index e8100a8f60..0b81c16170 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_health.js @@ -1,5 +1,5 @@ /* Copyright (C) 2016 NooBaa */ -/*eslint max-lines-per-function: ["error", 700]*/ +/*eslint max-lines-per-function: ['error', 700]*/ 'use strict'; @@ -8,6 +8,7 @@ const mocha = require('mocha'); const sinon = require('sinon'); const assert = require('assert'); const config = require('../../../config'); +const pkg = require('../../../package.json'); const fs_utils = require('../../util/fs_utils'); const nb_native = require('../../util/nb_native'); const { ConfigFS } = require('../../sdk/config_fs'); @@ -18,11 +19,21 @@ const { get_process_fs_context } = require('../../util/native_fs_utils'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli } = require('../system_tests/test_utils'); +const { CONFIG_DIR_UNLOCKED, CONFIG_DIR_LOCKED } = require('../../upgrade/nc_upgrade_manager'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_health'); const DEFAULT_FS_CONFIG = get_process_fs_context(); const bucket_storage_path = path.join(tmp_fs_path, 'bucket_storage_path'); +const os = require('os'); +const hostname = os.hostname(); + +const valid_system_json = { + [hostname]: { + 'current_version': pkg.version, + 'upgrade_history': { 'successful_upgrades': [] }, + }, +}; mocha.describe('nsfs nc health', function() { @@ -45,6 +56,10 @@ mocha.describe('nsfs nc health', function() { }); mocha.describe('nsfs nc health cli validations', function() { + mocha.afterEach(() => { + restore_health_if_needed(Health); + }); + mocha.it('https_port flag type validation - should fail', async function() { try { await exec_manage_cli(TYPES.DIAGNOSE, DIAGNOSE_ACTIONS.HEALTH, { 'http_port': '' }); @@ -137,7 +152,7 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, {config_root, ...account1_options}); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket1_options}); await fs_utils.file_must_exist(path.join(config_root, 'master_keys.json')); - const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage"); + const get_service_memory_usage = sinon.stub(Health, 'get_service_memory_usage'); get_service_memory_usage.onFirstCall().returns(Promise.resolve(100)); for (const user of Object.values(fs_users)) { await create_fs_user_by_platform(user.distinguished_name, user.distinguished_name, user.uid, user.gid); @@ -158,13 +173,20 @@ mocha.describe('nsfs nc health', function() { mocha.afterEach(async () => { await fs_utils.file_delete(config_fs.config_json_path); + restore_health_if_needed(Health); }); - mocha.it('Health all condition is success', async function() { - const get_service_state = sinon.stub(Health, "get_service_state"); + mocha.it('Health all condition is success1', async function() { + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_UNLOCKED + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); Health.all_account_details = true; Health.all_bucket_details = true; @@ -175,15 +197,16 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.checks.accounts_status.valid_accounts[0].name, 'account1'); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.valid_buckets[0].name, 'bucket1'); + assert_config_dir_status(health_status, valid_system_json.config_directory); + assert_upgrade_status(health_status); + await fs_utils.file_delete(Health.config_fs.system_json_path); }); mocha.it('NooBaa service is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'inactive', pid: 0 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); @@ -191,12 +214,10 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NooBaa endpoint return error response is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'MISSING_FORKS', total_fork_count: 3, running_workers: ['1', '3']}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); @@ -204,15 +225,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NSFS account with invalid storage path', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip invalid storage path check on the CLI const account_invalid_options = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid', nsfs_account_config: { new_buckets_path: path.join(new_buckets_path, '/invalid') } }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_options); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -223,23 +242,21 @@ mocha.describe('nsfs nc health', function() { mocha.it('NSFS bucket with invalid storage path', async function() { this.timeout(5000);// eslint-disable-line no-invalid-this - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); const resp = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account2_options }); const parsed_res = JSON.parse(resp).response.reply; // create it manually because we can not skip invalid storage path check on the CLI const bucket_invalid = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid', owner_account: parsed_res._id }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_invalid.name); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "STORAGE_NOT_EXIST"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'STORAGE_NOT_EXIST'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_invalid.name}); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account2_options.name}); }); @@ -250,22 +267,20 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_inaccessible_options.name); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_inaccessible_options.name}); @@ -277,18 +292,16 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with invalid owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account', owner_account: 'invalid_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "INVALID_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'INVALID_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); @@ -299,34 +312,30 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with empty owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "MISSING_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'MISSING_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); }); mocha.it('NSFS invalid bucket schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const bucket_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_schema', path: new_buckets_path }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_schema, 'invalid'); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -337,15 +346,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NSFS invalid account schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const account_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid_schema', path: new_buckets_path, bla: 5 }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_schema, 'invalid'); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -356,14 +363,12 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_account_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = false; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -374,14 +379,12 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_bucket_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -392,17 +395,15 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Config root path without bucket and account folders', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; Health.config_root = config_root_invalid; const old_config_fs = Health.config_fs; Health.config_fs = new ConfigFS(config_root_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); @@ -419,20 +420,18 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_options.name}); }); @@ -441,14 +440,12 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, debug: 5, ...account_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); @@ -466,14 +463,12 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); @@ -490,35 +485,31 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_dn_options.name }); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_dn_options.name); }); mocha.it('Account with inaccessible path - dn - NC_DISABLE_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); @@ -535,14 +526,12 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn - NC_DISABLE_HEALTH_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); @@ -559,14 +548,12 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with invalid dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...invalid_account_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: invalid_account_dn_options.name }); @@ -576,20 +563,18 @@ mocha.describe('nsfs nc health', function() { const found = health_status.checks.accounts_status.invalid_accounts.filter(account => account.name === invalid_account_dn_options.name); assert.strictEqual(found.length, 1); - assert.strictEqual(found[0].code, "INVALID_DISTINGUISHED_NAME"); + assert.strictEqual(found[0].code, 'INVALID_DISTINGUISHED_NAME'); }); mocha.it('Account with new_buckets_path missing and allow_bucket_creation false, valid account', async function() { const account_valid = { name: 'account_valid', uid: 999, gid: 999, allow_bucket_creation: false }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_valid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_valid.name }); @@ -601,19 +586,82 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with new_buckets_path missing and allow_bucket_creation true, invalid account', async function() { const account_invalid = { name: 'account_invalid', uid: 999, gid: 999, allow_bucket_creation: true }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_invalid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); + const get_service_state = sinon.stub(Health, 'get_service_state'); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + const get_endpoint_response = sinon.stub(Health, 'get_endpoint_response'); get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_invalid.name }); }); + + mocha.it('Health all condition - invalid config directory upgrade', async function() { + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + last_failure: { error: 'mock error'} + } + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + assert_config_dir_status(health_status, valid_system_json.config_directory); + assert_upgrade_status(health_status, valid_system_json.config_directory.upgrade_history.last_failure); + await fs_utils.file_delete(Health.config_fs.system_json_path); + }); + + mocha.it('Health all condition - valid ongoing config directory upgrade', async function() { + valid_system_json.config_directory = { + config_dir_version: config_fs.config_dir_version, + upgrade_package_version: pkg.version, + phase: CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + }, + in_progress_upgrade: { from_version: 'x', to_version: 'y'} + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + assert_config_dir_status(health_status, valid_system_json.config_directory); + assert_upgrade_status(health_status, valid_system_json.config_directory.upgrade_history.last_failure); + await fs_utils.file_delete(Health.config_fs.system_json_path); + }); }); }); + +/** + * assert_config_dir_status asserts config directory status + * @param {Object} health_status + * @param {Object} expected_config_dir + */ +function assert_config_dir_status(health_status, expected_config_dir) { + const actual_config_dir_health = health_status.checks.config_directory_status; + assert.strictEqual(actual_config_dir_health.phase, expected_config_dir.phase); + assert.deepStrictEqual(actual_config_dir_health.in_progress_upgrade, expected_config_dir.in_progress_upgrade); + assert.strictEqual(actual_config_dir_health.config_dir_version, expected_config_dir.config_dir_version); + assert.strictEqual(actual_config_dir_health.upgrade_package_version, expected_config_dir.upgrade_package_version); +} + +/** + * assert_upgrade_status asserts there if there is an upgrade error + * @param {Object} health_status + * @param {Object} [config_directory_last_failure=undefined] + */ +function assert_upgrade_status(health_status, config_directory_last_failure = undefined) { + assert.deepStrictEqual(health_status.checks.upgrade_status, config_directory_last_failure); +} + +/** + * restore_health_if_needed restores health obj functions if needed + * @param {*} health_obj + */ +function restore_health_if_needed(health_obj) { + if (health_obj?.get_service_state?.restore) health_obj.get_service_state.restore(); + if (health_obj?.get_endpoint_response?.restore) health_obj.get_endpoint_response.restore(); +} diff --git a/src/upgrade/nc_upgrade_manager.js b/src/upgrade/nc_upgrade_manager.js index 83bde5fa57..e0630fe94b 100644 --- a/src/upgrade/nc_upgrade_manager.js +++ b/src/upgrade/nc_upgrade_manager.js @@ -6,10 +6,9 @@ const _ = require('lodash'); const path = require('path'); const util = require('util'); const pkg = require('../../package.json'); -const dbg = require('../util/debug_module')('UPGRADE'); +const dbg = require('../util/debug_module')(__filename); const { should_upgrade, run_upgrade_scripts, version_compare } = require('./upgrade_utils'); -dbg.set_process_name('Upgrade'); const hostname = os.hostname(); const CONFIG_DIR_LOCKED = 'CONFIG_DIR_LOCKED'; diff --git a/src/upgrade/upgrade_utils.js b/src/upgrade/upgrade_utils.js index 453548eb1c..ff34af4d15 100644 --- a/src/upgrade/upgrade_utils.js +++ b/src/upgrade/upgrade_utils.js @@ -4,9 +4,7 @@ const fs = require('fs'); const _ = require('lodash'); const path = require('path'); -const dbg = require('../util/debug_module')('UPGRADE'); -dbg.set_process_name('Upgrade'); - +const dbg = require('../util/debug_module')(__filename); /** * @param {string} ver