diff --git a/src/agent/block_store_speed.js b/src/agent/block_store_speed.js index b3580732ee..b2cd733b16 100644 --- a/src/agent/block_store_speed.js +++ b/src/agent/block_store_speed.js @@ -4,7 +4,6 @@ // const _ = require('lodash'); const argv = require('minimist')(process.argv); const cluster = require('cluster'); -const mongodb = require('mongodb'); const api = require('../api'); const config = require('../../config'); @@ -12,6 +11,7 @@ const dotenv = require('../util/dotenv'); const Speedometer = require('../util/speedometer'); const { RPC_BUFFERS } = require('../rpc'); +const ObjectID = require('../util/objectid'); dotenv.load(); argv.email = argv.email || 'demo@noobaa.com'; @@ -60,7 +60,7 @@ async function worker(client) { } async function write_block(client) { - const block_id = new mongodb.ObjectId(); + const block_id = new ObjectID(null); return client.block_store.write_block({ [RPC_BUFFERS]: { data: Buffer.allocUnsafe(argv.size) }, block_md: { diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index 2f5766679c..5cf3d3fffa 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -11,7 +11,7 @@ const nb_native = require('../util/nb_native'); const { ConfigFS } = require('../sdk/config_fs'); const cloud_utils = require('../util/cloud_utils'); const native_fs_utils = require('../util/native_fs_utils'); -const mongo_utils = require('../util/mongo_utils'); +const ObjectID = require('../util/objectid'); const SensitiveString = require('../util/sensitive_string'); const { account_id_cache } = require('../sdk/accountspace_fs'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; @@ -169,7 +169,7 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) { * @returns { Promise<{ code: ManageCLIResponse.BucketCreated, detail: Object, event_arg: Object }>} */ async function add_bucket(data) { - data._id = mongo_utils.mongoObjectId(); + data._id = new ObjectID(null); const parsed_bucket_data = await config_fs.create_bucket_config_file(data); await set_bucker_owner(parsed_bucket_data); return { code: ManageCLIResponse.BucketCreated, detail: parsed_bucket_data, event_arg: { bucket: data.name }}; @@ -413,7 +413,7 @@ async function fetch_existing_account_data(action, target, decrypt_secret_key) { * @returns { Promise<{ code: typeof ManageCLIResponse.AccountCreated, detail: Object, event_arg: Object }>} */ async function add_account(data) { - data._id = mongo_utils.mongoObjectId(); + data._id = new ObjectID(null); await config_fs.create_account_config_file(data); return { code: ManageCLIResponse.AccountCreated, detail: data, event_arg: { account: data.name } }; } diff --git a/src/nc/nc_utils.js b/src/nc/nc_utils.js index 74f6ac40fa..98a8b4a8b1 100644 --- a/src/nc/nc_utils.js +++ b/src/nc/nc_utils.js @@ -1,8 +1,7 @@ /* Copyright (C) 2024 NooBaa */ 'use strict'; -const mongo_utils = require('../util/mongo_utils'); - +const objectid = require('../util/objectid'); /** * generate_id will generate an id that we use to identify entities (such as account, bucket, etc.). */ @@ -11,7 +10,7 @@ const mongo_utils = require('../util/mongo_utils'); // - this function implantation should be db_client.new_object_id(), // but to align with manage nsfs we won't change it now function generate_id() { - return mongo_utils.mongoObjectId(); + return objectid(); } /** diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index e3d3788b2b..eb414a100a 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -1,6 +1,7 @@ /* Copyright (C) 2020 NooBaa */ 'use strict'; +const objectid = require('../util/objectid'); const _ = require('lodash'); const util = require('util'); const path = require('path'); @@ -9,7 +10,6 @@ const config = require('../../config'); const RpcError = require('../rpc/rpc_error'); const js_utils = require('../util/js_utils'); const nb_native = require('../util/nb_native'); -const mongo_utils = require('../util/mongo_utils'); const KeysSemaphore = require('../util/keys_semaphore'); const { get_umasked_mode, @@ -314,7 +314,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { new_bucket_defaults(account, { name, tag, lock_enabled, force_md5_etag }, create_uls, bucket_storage_path) { return { - _id: mongo_utils.mongoObjectId(), + _id: objectid(), name, tag: js_utils.default_value(tag, undefined), owner_account: account._id, diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index a98071294a..10541e4781 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -5,7 +5,9 @@ import * as mongodb from 'mongodb'; import { EventEmitter } from 'events'; import { Readable, Writable } from 'stream'; import { IncomingMessage, ServerResponse } from 'http'; -import { ObjectPart, Checksum} from '@aws-sdk/client-s3'; +import { ObjectPart, Checksum } from '@aws-sdk/client-s3'; + +import ObjectID = require("../util/objectid"); type Semaphore = import('../util/semaphore'); type KeysSemaphore = import('../util/keys_semaphore'); @@ -39,6 +41,8 @@ type ReplicationLogCandidates = Record }; + + interface MapByID { [id: string]: T } interface Base { @@ -46,7 +50,7 @@ interface Base { toString?(): string; } -type ID = mongodb.ObjectID; +type ID = ObjectID; type DBBuffer = mongodb.Binary | Buffer; interface System extends Base { @@ -720,8 +724,8 @@ interface DBClient { populate(docs: object[] | object, doc_path: string, collection: DBCollection, fields: object): Promise; resolve_object_ids_recursive(idmap: object, item: object): object; resolve_object_ids_paths(idmap: object, item: object, paths: string[], allow_missing: boolean): object; - new_object_id(): mongodb.ObjectId; - parse_object_id(id_str: string): mongodb.ObjectId; + new_object_id(): ObjectID; + parse_object_id(id_str: string): ObjectID; fix_id_type(doc: object[] | object): object[] | object; is_object_id(id: object[] | object): boolean; is_err_duplicate_key(err: object): boolean; @@ -1150,4 +1154,4 @@ interface GetObjectAttributesParts { MaxParts?: number; IsTruncated?: boolean; Parts?: ObjectPart[]; - } \ No newline at end of file +} \ No newline at end of file diff --git a/src/sdk/objectid.d.ts b/src/sdk/objectid.d.ts new file mode 100644 index 0000000000..53a94567d0 --- /dev/null +++ b/src/sdk/objectid.d.ts @@ -0,0 +1,32 @@ +/* Copyright (C) 2016 NooBaa */ + +/** + * Represents a BSON ObjectID type. + */ +declare interface ObjectID { + readonly id: string | Buffer; + readonly str: string; + + toHexString(): string; + equals(other: ObjectID | string): boolean; + getTimestamp(): Date; + generate(time?: number): string; + toJSON(): string; + toString(): string; +} + +/** + * Constructor and utility functions for the ObjectID type. + */ +declare interface ObjectIDConstructor { + new(id?: string | number | Buffer): ObjectID; + isValid(id: string | Buffer | ObjectID): boolean; + (id?: string | number | Buffer): ObjectID; +} + +/** + * The ObjectID constructor and utilities. + */ +declare const ObjectID: ObjectIDConstructor; + +export = ObjectID; diff --git a/src/server/analytic_services/activity_log_store.js b/src/server/analytic_services/activity_log_store.js index c0411730b9..87737af807 100644 --- a/src/server/analytic_services/activity_log_store.js +++ b/src/server/analytic_services/activity_log_store.js @@ -1,10 +1,10 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); const _ = require('lodash'); const db_client = require('../../util/db_client'); +const ObjectID = require('../../util/objectid'); const P = require('../../util/promise'); const activity_log_schema = require('./activity_log_schema'); const activity_log_indexes = require('./activity_log_indexes'); @@ -25,7 +25,7 @@ class ActivityLogStore { } make_activity_log_id(id_str) { - return new mongodb.ObjectID(id_str); + return new ObjectID(id_str); } diff --git a/src/server/analytic_services/history_data_store.js b/src/server/analytic_services/history_data_store.js index 3ab9068cec..0ff8efcda0 100644 --- a/src/server/analytic_services/history_data_store.js +++ b/src/server/analytic_services/history_data_store.js @@ -1,8 +1,6 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); - // const dbg = require('../../util/debug_module')(__filename); const config = require('../../../config.js'); // const pkg = require('../../../package.json'); @@ -11,6 +9,7 @@ const P = require('../../util/promise'); const db_client = require('../../util/db_client'); const system_history_schema = require('../analytic_services/system_history_schema'); +const ObjectID = require('../../util/objectid'); class HistoryDataStore { constructor() { @@ -30,7 +29,7 @@ class HistoryDataStore { const time_stamp = new Date(); const record_expiration_date = new Date(time_stamp.getTime() - config.STATISTICS_COLLECTOR_EXPIRATION); const record = { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), time_stamp, system_snapshot: item, history_type: 'SYSTEM' diff --git a/src/server/func_services/func_stats_store.js b/src/server/func_services/func_stats_store.js index 927ad0d972..17c8dd4e83 100644 --- a/src/server/func_services/func_stats_store.js +++ b/src/server/func_services/func_stats_store.js @@ -2,8 +2,7 @@ 'use strict'; // const _ = require('lodash'); -const mongodb = require('mongodb'); - +const ObjectID = require('../../util/objectid'); // const dbg = require('../../util/debug_module')(__filename); const db_client = require('../../util/db_client'); @@ -26,7 +25,7 @@ class FuncStatsStore { } make_func_stat_id(id_str) { - return new mongodb.ObjectId(id_str); + return new ObjectID(id_str); } async create_func_stat(stat) { diff --git a/src/server/func_services/func_store.js b/src/server/func_services/func_store.js index 1313a8eb92..f1e84abffc 100644 --- a/src/server/func_services/func_store.js +++ b/src/server/func_services/func_store.js @@ -1,13 +1,11 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); - const db_client = require('../../util/db_client'); const func_schema = require('./func_schema'); const func_indexes = require('./func_indexes'); - +const ObjectID = require('../../util/objectid'); class FuncStore { constructor() { @@ -24,7 +22,7 @@ class FuncStore { } make_func_id(id_str) { - return new mongodb.ObjectId(id_str); + return new ObjectID(id_str); } async create_func(func) { diff --git a/src/server/node_services/nodes_store.js b/src/server/node_services/nodes_store.js index 3b54631aa1..f3b1509815 100644 --- a/src/server/node_services/nodes_store.js +++ b/src/server/node_services/nodes_store.js @@ -2,13 +2,12 @@ 'use strict'; const _ = require('lodash'); -const mongodb = require('mongodb'); const dbg = require('../../util/debug_module')(__filename); const node_schema = require('./node_schema'); const db_client = require('../../util/db_client'); const P = require('../../util/promise'); - +const ObjectID = require('../../util/objectid'); class NodesStore { constructor(test_suffix = '') { @@ -24,7 +23,7 @@ class NodesStore { } make_node_id(id_str) { - return new mongodb.ObjectId(id_str); + return new ObjectID(id_str); } is_connected() { diff --git a/src/server/notifications/alerts_log_store.js b/src/server/notifications/alerts_log_store.js index 77d7aa915e..ff4460b73f 100644 --- a/src/server/notifications/alerts_log_store.js +++ b/src/server/notifications/alerts_log_store.js @@ -1,13 +1,15 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); const _ = require('lodash'); const P = require('../../util/promise'); const db_client = require('../../util/db_client'); const alerts_log_schema = require('./alerts_log_schema'); +// @ts-ignore +const ObjectID = require('../../util/objectid'); + class AlertsLogStore { constructor() { @@ -22,8 +24,8 @@ class AlertsLogStore { return AlertsLogStore._instance; } - make_alert_log_id(id_str) { - return new mongodb.ObjectID(id_str); + make_alert_log_id() { + return new ObjectID(null); } create(alert_log) { @@ -94,12 +96,12 @@ class AlertsLogStore { let _id; if (ids) { - const obj_ids = ids.map(id => new mongodb.ObjectID(id)); + const obj_ids = ids.map(id => new ObjectID(id)); _id = { $in: obj_ids }; } else if (till) { - _id = { $lt: new mongodb.ObjectID(till) }; + _id = { $lt: new ObjectID(till) }; } else if (since) { - _id = { $gt: new mongodb.ObjectID(since) }; + _id = { $gt: new ObjectID(since) }; } return _.omitBy({ diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 90cd1ad68c..b00a907445 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -6,9 +6,8 @@ const _ = require('lodash'); const assert = require('assert'); const moment = require('moment'); -const mongodb = require('mongodb'); const mime = require('mime'); - +const ObjectID = require('../../util/objectid'); const P = require('../../util/promise'); const dbg = require('../../util/debug_module')(__filename); const db_client = require('../../util/db_client'); @@ -80,18 +79,18 @@ class MDStore { if (zero_suffix) { suffix = '0'.repeat(16); } else { - suffix = String(new mongodb.ObjectId()).slice(8, 24); + suffix = String(new ObjectID(null)).slice(8, 24); } const hex_id = padded_hex_time + suffix; assert(padded_hex_time.length === 8); assert(suffix.length === 16); assert(hex_id.length === 24); assert(parseInt(padded_hex_time, 16) === Math.floor(time / 1000)); - return new mongodb.ObjectId(hex_id); + return new ObjectID(hex_id); } is_valid_md_id(id_str) { - return mongodb.ObjectId.isValid(id_str); + return ObjectID.isValid(id_str); } ///////////// @@ -1869,7 +1868,7 @@ function sort_list_uploads_with_delimiter(a, b) { * @returns {nb.ID} */ function make_md_id(id_str) { - return new mongodb.ObjectId(id_str); + return new ObjectID(id_str); } diff --git a/src/server/system_services/config_file_store.js b/src/server/system_services/config_file_store.js index 7a9ab7d435..0a556c9253 100644 --- a/src/server/system_services/config_file_store.js +++ b/src/server/system_services/config_file_store.js @@ -4,10 +4,9 @@ const _ = require('lodash'); const dbg = require('../../util/debug_module')(__filename); const db_client = require('../../util/db_client'); -const mongodb = require('mongodb'); const config_file_schema = require('./schemas/config_file_schema'); const config_file_indexes = require('./schemas/config_file_indexes'); - +const ObjectID = require('../../util/objectid'); class ConfigFileStore { constructor() { @@ -26,7 +25,7 @@ class ConfigFileStore { async insert(item) { dbg.log0(`insert`, item); _.defaults(item, { - _id: new mongodb.ObjectId() + _id: new ObjectID(null) }); // There shouldn't be more than one record, this is being on the safe side this._config_files.validate(item); diff --git a/src/server/system_services/replication_store.js b/src/server/system_services/replication_store.js index d42343a4d6..b350e31dda 100644 --- a/src/server/system_services/replication_store.js +++ b/src/server/system_services/replication_store.js @@ -2,11 +2,10 @@ 'use strict'; const _ = require('lodash'); -const mongodb = require('mongodb'); const db_client = require('../../util/db_client'); const dbg = require('../../util/debug_module')(__filename); const replication_schema = require('./schemas/replication_configuration_schema'); - +const ObjectID = require('../../util/objectid'); class ReplicationStore { constructor() { @@ -25,7 +24,7 @@ class ReplicationStore { item = _.omitBy(item, _.isNil); dbg.log1(`insert_replication`, item); const record = { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), ...item }; this._replicationconfigs.validate(record); diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js index 5142207789..a278b1e320 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js @@ -1836,6 +1836,7 @@ describe('cli account flow distinguished_name - permissions', function() { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, { uid: 0, gid: 0 }, 0o700); const res = await exec_manage_cli(type, ACTIONS.ADD, accounts.root.cli_options); + console.log("res ======================", res); assert_account(JSON.parse(res).response.reply, accounts.root.cli_options, false); }, timeout); diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq b/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq index 346b2f4e35..72e691d7ac 100644 --- a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq +++ b/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq @@ -13,8 +13,7 @@ Authorization: AWS 123:Zy/+Do9VcaCZcfdno7lXzjw6qHM= const _ = require('lodash'); const util = require('util'); -const mongodb = require('mongodb'); -const mongoose = require('mongoose'); +const ObjectID = require('../../../../util/objectid'); const P = require('./promise'); const RpcError = require('../rpc/rpc_error'); @@ -108,7 +107,7 @@ function populate(docs, doc_path, collection, fields) { function resolve_object_ids_recursive(idmap, item) { _.each(item, (val, key) => { - if (val instanceof mongodb.ObjectId) { + if (val instanceof ObjectID) { if (key !== '_id') { const obj = idmap[val]; if (obj) { @@ -144,7 +143,7 @@ function resolve_object_ids_paths(idmap, item, paths, allow_missing) { } function make_object_id(id_str) { - return new mongodb.ObjectId(id_str); + return new ObjectID(id_str); } function fix_id_type(doc) { @@ -157,12 +156,10 @@ function fix_id_type(doc) { } // apparently mongoose defined it's own class of ObjectID -// instead of using the class from mongodb driver, // so we have to check both for now, // until we can get rid of mongoose completely. function is_object_id(id) { - return (id instanceof mongodb.ObjectId) || - (id instanceof mongoose.Types.ObjectId); + return (id instanceof ObjectID); } function is_err_duplicate_key(err) { diff --git a/src/test/unit_tests/test_agent_blocks_reclaimer.js b/src/test/unit_tests/test_agent_blocks_reclaimer.js index 02d6c87f33..04095bb24a 100644 --- a/src/test/unit_tests/test_agent_blocks_reclaimer.js +++ b/src/test/unit_tests/test_agent_blocks_reclaimer.js @@ -4,13 +4,11 @@ // setup coretest first to prepare the env const coretest = require('./coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); - +const ObjectID = require('../../util/objectid'); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const mongodb = require('mongodb'); - const P = require('../../util/promise'); const config = require('../../../config'); const db_client = require('../../util/db_client'); @@ -246,13 +244,13 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should mark reclaimed on deleted nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), rpc_address: 'n2n://SlothTown', online: false, deleted: new Date() }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), node: nodes[0]._id, deleted: new Date() }]; @@ -271,12 +269,12 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should not mark reclaimed on offline nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), rpc_address: 'n2n://SlothTown', online: false, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), node: nodes[0]._id, deleted: new Date(), fail_to_delete: true @@ -296,14 +294,14 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should mark reclaimed on non existing nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), rpc_address: 'n2n://SlothTown', online: true, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), // Non existing node on purpose - node: new mongodb.ObjectId(), + node: new ObjectID(null), deleted: new Date() }]; const reclaimer_mock = @@ -321,16 +319,16 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should not mark reclaimed on failure to delete', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), rpc_address: 'n2n://SlothTown', online: true, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), node: nodes[0]._id, deleted: new Date() }, { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), node: nodes[0]._id, deleted: new Date(), fail_to_delete: true diff --git a/src/test/unit_tests/test_agent_blocks_verifier.js b/src/test/unit_tests/test_agent_blocks_verifier.js index 9c3c91ea74..6ecc4b8476 100644 --- a/src/test/unit_tests/test_agent_blocks_verifier.js +++ b/src/test/unit_tests/test_agent_blocks_verifier.js @@ -14,10 +14,9 @@ const P = require('../../util/promise'); const AgentBlocksVerifier = require('../../server/bg_services/agent_blocks_verifier').AgentBlocksVerifier; const db_client = require('../../util/db_client'); const schema_utils = require('../../util/schema_utils'); -const mongodb = require('mongodb'); const config = require('../../../config'); const { ChunkDB, BlockDB } = require('../../server/object_services/map_db_types'); - +const ObjectID = require('../../util/objectid'); class VerifierMock extends AgentBlocksVerifier { /** * @@ -122,15 +121,15 @@ class VerifierMock extends AgentBlocksVerifier { mocha.describe('mocked agent_blocks_verifier', function() { - const tier_id = new mongodb.ObjectId(); - const bucket_id = new mongodb.ObjectId(); - const system_id = new mongodb.ObjectId(); + const tier_id = new ObjectID(null); + const bucket_id = new ObjectID(null); + const system_id = new ObjectID(null); mocha.it('should verify blocks on nodes', function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [make_node('bla2', false)]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), chunk_coder_config: { frag_digest_type: 'sloth_type' } @@ -154,7 +153,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [make_node('bla1', true)]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), chunk_coder_config: { frag_digest_type: 'sloth_type' } @@ -179,7 +178,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { const self = this; // eslint-disable-line no-invalid-this // const nodes = [make_node('node1')]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), system: system_id, chunk_coder_config: { frag_digest_type: 'sloth_type' @@ -187,7 +186,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { }]; const chunks = [make_schema_chunk(chunk_coder_configs[0]._id, [make_schema_frag()])]; const pools = [make_schema_pool('pool1')]; - const blocks = [make_schema_block(chunks[0].frags[0]._id, chunks[0]._id, new mongodb.ObjectId(), pools[0]._id)]; + const blocks = [make_schema_block(chunks[0].frags[0]._id, chunks[0]._id, new ObjectID(null), pools[0]._id)]; const verifier_mock = new VerifierMock(blocks, [], chunks, pools); return P.resolve() @@ -209,7 +208,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_block(frag_id, chunk_id, node_id, pool_id) { return { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), node: node_id, frag: frag_id, chunk: chunk_id, @@ -226,7 +225,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_frag() { return { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), digest: Buffer.from('bla') }; } @@ -237,7 +236,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_chunk(cc_id, frags) { return { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), system: system_id, bucket: bucket_id, tier: tier_id, @@ -263,7 +262,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_node(node_name, offline) { return { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), name: node_name, pool: 'pool1', node_type: 'BLOCK_STORE_FS', @@ -293,7 +292,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_pool(name) { return { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), name: name, system: undefined, resource_type: 'HOSTS', diff --git a/src/test/unit_tests/test_lifecycle.js b/src/test/unit_tests/test_lifecycle.js index 80d468d149..4b6f60fa6b 100644 --- a/src/test/unit_tests/test_lifecycle.js +++ b/src/test/unit_tests/test_lifecycle.js @@ -8,9 +8,8 @@ const { NodeHttpHandler } = require("@smithy/node-http-handler"); const util = require('util'); const mocha = require('mocha'); const assert = require('assert'); -const mongodb = require('mongodb'); const { v4: uuid } = require('uuid'); - +const ObjectID = require('../../util/objectid'); const P = require('../../util/promise'); const config = require('../../../config'); const MDStore = require('../../server/object_services/md_store').MDStore; @@ -108,7 +107,7 @@ mocha.describe('lifecycle', () => { if (tagging) update.tagging = tagging; console.log('create_mock_object bucket', bucket, 'key', key, 'update', util.inspect(update)); - const id = new mongodb.ObjectId(obj_id); + const id = new ObjectID(obj_id); console.log('create_mock_object id', id, 'obj_id', obj_id); const updateResult = await MDStore.instance().update_object_by_id(id, update); diff --git a/src/test/unit_tests/test_map_reader.js b/src/test/unit_tests/test_map_reader.js index 188681fc0a..3419610094 100644 --- a/src/test/unit_tests/test_map_reader.js +++ b/src/test/unit_tests/test_map_reader.js @@ -9,8 +9,7 @@ coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); // const util = require('util'); const mocha = require('mocha'); // const assert = require('assert'); -const mongodb = require('mongodb'); - +const ObjectID = require('../../util/objectid'); // const P = require('../../util/promise'); // const MDStore = require('../../server/object_services/md_store').MDStore; // const map_writer = require('../../server/object_services/map_writer'); @@ -40,14 +39,14 @@ coretest.describe_mapper_test_case({ // TODO test_map_reader mocha.it('read_object_mapping', function() { - const obj = { size: 100, _id: new mongodb.ObjectId() }; + const obj = { size: 100, _id: new ObjectID(null) }; const start = 0; const end = 100; return map_reader.read_object_mapping(obj, start, end); }); mocha.it('read_object_mapping_admin', function() { - const obj = { size: 100, _id: new mongodb.ObjectId() }; + const obj = { size: 100, _id: new ObjectID(null) }; const skip = 0; const limit = 100; return map_reader.read_object_mapping_admin(obj, skip, limit); diff --git a/src/test/unit_tests/test_mapper.js b/src/test/unit_tests/test_mapper.js index 9581a41468..60d03daf49 100644 --- a/src/test/unit_tests/test_mapper.js +++ b/src/test/unit_tests/test_mapper.js @@ -5,16 +5,14 @@ // setup coretest first to prepare the env const coretest = require('./coretest'); coretest.no_setup(); - +const ObjectID = require('../../util/objectid'); const _ = require('lodash'); const util = require('util'); const mocha = require('mocha'); const assert = require('assert'); -const mongodb = require('mongodb'); const config = require('../../../config.js'); const mapper = require('../../server/object_services/mapper'); - coretest.describe_mapper_test_case({ name: 'mapper', }, ({ @@ -32,45 +30,45 @@ coretest.describe_mapper_test_case({ }) => { const frags = _.concat( - _.times(data_frags, data_index => ({ _id: new mongodb.ObjectId(), data_index })), - _.times(parity_frags, parity_index => ({ _id: new mongodb.ObjectId(), parity_index })) + _.times(data_frags, data_index => ({ _id: new ObjectID(null), data_index })), + _.times(parity_frags, parity_index => ({ _id: new ObjectID(null), parity_index })) ); - const first_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'first_pool' + i, })); - const second_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'second_pool' + i, })); - const external_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'external_pool' + i, })); + const first_pools = _.times(num_pools, i => ({ _id: new ObjectID(null), name: 'first_pool' + i, })); + const second_pools = _.times(num_pools, i => ({ _id: new ObjectID(null), name: 'second_pool' + i, })); + const external_pools = _.times(num_pools, i => ({ _id: new ObjectID(null), name: 'external_pool' + i, })); const pool_by_id = _.keyBy(_.concat(first_pools, second_pools, external_pools), '_id'); const first_mirrors = data_placement === 'MIRROR' ? first_pools.map(pool => ({ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), spread_pools: [pool] })) : [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), spread_pools: first_pools }]; const second_mirrors = data_placement === 'MIRROR' ? second_pools.map(pool => ({ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), spread_pools: [pool] })) : [{ - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), spread_pools: second_pools }]; const first_tier = { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), name: 'first_tier', data_placement, mirrors: first_mirrors, chunk_config: { chunk_coder_config }, }; const second_tier = { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), name: 'second_tier', data_placement, mirrors: second_mirrors, chunk_config: { chunk_coder_config }, }; const tiering = { - _id: new mongodb.ObjectId(), + _id: new ObjectID(null), name: 'tiering_policy', tiers: [{ order: 0, @@ -653,7 +651,7 @@ coretest.describe_mapper_test_case({ const pool = params.pool || pools_to_use[pool_i]; const pool_name = pool.name; - const _id = new mongodb.ObjectID(); + const _id = new ObjectID(null); const _id_str = _id.toString(); return { diff --git a/src/test/unit_tests/test_schema_keywords.js b/src/test/unit_tests/test_schema_keywords.js index 34bd5a864a..becc517e25 100644 --- a/src/test/unit_tests/test_schema_keywords.js +++ b/src/test/unit_tests/test_schema_keywords.js @@ -5,8 +5,8 @@ const mocha = require('mocha'); const { default: Ajv } = require('ajv'); const schema_keywords = require('../../util/schema_keywords'); const SensitiveString = require('../../util/sensitive_string'); -const mongodb = require('mongodb'); const assert = require('assert'); +const ObjectID = require('../../util/objectid'); /** * @typedef {import('ajv').KeywordCxt} KeywordCxt @@ -75,7 +75,7 @@ mocha.describe('Test Schema Keywords', function() { mocha.it('Test keyword objectid', async function() { const validator = ajv.getSchema('test_schema_keywords#/methods/params'); - const should_pass = { key3: new mongodb.ObjectId() }; + const should_pass = { key3: new ObjectID(null) }; assert.strictEqual(validator(should_pass), true); const should_fail = { key3: 'not_an_objectid' }; assert.strictEqual(validator(should_fail), false); diff --git a/src/util/db_client.js b/src/util/db_client.js index 78bcee9d5e..66d4e11117 100644 --- a/src/util/db_client.js +++ b/src/util/db_client.js @@ -2,14 +2,13 @@ /** @typedef {typeof import('../sdk/nb')} nb */ 'use strict'; -const mongodb = require('mongodb'); const { EventEmitter } = require('events'); const dbg = require('./debug_module')(__filename); const config = require('../../config'); const mongo_client = require('./mongo_client'); const postgres_client = require('./postgres_client'); - +const ObjectID = require('../util/objectid'); /** * A simple noop db client for cases where we run without a DB. * @implements {nb.DBClient} @@ -36,8 +35,8 @@ class NoneDBClient extends EventEmitter { async populate(docs, doc_path, collection, fields) { return this.noop(); } resolve_object_ids_recursive(idmap, item) { return this.noop(); } resolve_object_ids_paths(idmap, item, paths, allow_missing) { return this.noop(); } - new_object_id() { return new mongodb.ObjectId(); } - parse_object_id(id_str) { return new mongodb.ObjectId(String(id_str || undefined)); } + new_object_id() { return new ObjectID(null); } + parse_object_id(id_str) { return new ObjectID(String(id_str || undefined)); } fix_id_type(doc) { return doc; } is_object_id(id) { return false; } is_err_duplicate_key(err) { return false; } diff --git a/src/util/objectid.js b/src/util/objectid.js new file mode 100644 index 0000000000..e3f8ca5b2a --- /dev/null +++ b/src/util/objectid.js @@ -0,0 +1,144 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + +const crypto = require('crypto'); +const os = require('os'); +const dbg = require('./debug_module')(__filename); + +let index = Math.floor(Math.random() * 0xFFFFFF); + +// Process ID, with fallback for environments that don't have process.pid (e.g., browser) +const process_id = (typeof process === 'undefined' || + typeof process.pid !== 'number' ? Math.floor(Math.random() * 100000) : process.pid) % 0xFFFF; + +const check_for_hex_reg_exp = /^[0-9a-fA-F]{24}$/; + +/** + * Generate the machine ID using the hostname. + * @returns {number} A 3-byte identifier as a number. + */ +const get_machine_id = (() => { + let machineId; + return () => { + if (!machineId) { + const hostname = os.hostname(); + const hash = crypto.createHash('md5').update(hostname).digest(); + machineId = hash.readUIntBE(0, 3); // Use first 3 bytes of MD5 hash of the hostname + } + return machineId; + }; +})(); + +function next() { + index = (index + 1) % 0xFFFFFF; + return index; +} + +/** + * Represents a BSON ObjectID type. + * @class + * @param {string | number | Buffer | ObjectID} id - The input value to construct the ObjectID. + */ +function ObjectID(id) { + if (!(this instanceof ObjectID)) return new ObjectID(id); + if (id instanceof ObjectID) return id; + + this._bsontype = 'ObjectID'; + + let __id; + if (id === null || typeof id === 'number' || id === undefined) { + // @ts-ignore + __id = this.generate(id); // we are making sure that it is called on in case of null or number + } else if (typeof id === 'string' && id.length === 24 && check_for_hex_reg_exp.test(id)) { + __id = Buffer.from(id, 'hex'); + } else if (Buffer.isBuffer(id) && id.length === 12) { + __id = Buffer.from(id); + } else { + dbg.error("Invalid input: Must be a 24-character hex string, 12-byte Buffer, or number id = ", id); + throw new Error('Invalid input: Must be a 24-character hex string, 12-byte Buffer, or number. '); + } + + this.id = __id; + return this; +} + +/** + * Generate a 12-byte buffer for the ObjectID. + * @param {number} [time] - Optional timestamp to use in the generation. + * @returns {Buffer} The generated 12-byte buffer. + */ +ObjectID.prototype.generate = function(time) { + if (!time || typeof time !== 'number') { + time = Math.floor(Date.now() / 1000); // Current time in seconds + } + + const buffer = Buffer.alloc(12); + + // 4-byte timestamp + buffer.writeUInt32BE(time, 0); + + // 3-byte machine identifier + buffer.writeUIntBE(get_machine_id(), 4, 3); + + // 2-byte process ID + buffer.writeUInt16BE(process_id, 7); + + // 3-byte counter + const inc = next(); + buffer.writeUIntBE(inc, 9, 3); + + return buffer; +}; + +ObjectID.prototype.toHexString = function() { + return this.id.toString('hex'); +}; + +ObjectID.prototype.getTimestamp = function() { + const timestamp = this.id.readUInt32BE(0); + return new Date(timestamp * 1000); // Convert seconds to milliseconds +}; + +/** + * Compare equality with another ObjectID. + * @param {ObjectID | string} otherId - The other ObjectID to compare. + * @returns {boolean} True if equal, otherwise false. + */ +ObjectID.prototype.equals = function(otherId) { + if (otherId instanceof ObjectID) { + return this.toHexString() === otherId.toHexString(); + } else if (typeof otherId === 'string' && check_for_hex_reg_exp.test(otherId)) { + return this.toHexString() === otherId.toLowerCase(); + } + return false; +}; + +ObjectID.isValid = function(id) { + if (!id) return false; + + if (typeof id === 'string') { + return id.length === 24 && check_for_hex_reg_exp.test(id); + } + + if (Buffer.isBuffer(id)) { + return id.length === 12; + } + + if (id instanceof ObjectID) { + return true; + } + + return false; +}; + +//For better logging presentation +const inspect = Symbol.for('nodejs.util.inspect.custom'); +ObjectID.prototype[inspect] = function() { + return `ObjectID("${this.toHexString()}")`; +}; + +ObjectID.prototype.toJSON = ObjectID.prototype.toHexString; +ObjectID.prototype.toString = ObjectID.prototype.toHexString; + +module.exports = ObjectID; +ObjectID.default = ObjectID; diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index da80b2da1b..5b52a051c5 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -30,6 +30,9 @@ const config = require('../../config'); const ssl_utils = require('./ssl_utils'); const DB_CONNECT_ERROR_MESSAGE = 'Could not acquire client from DB connection pool'; + +const ObjectID = require('../util/objectid'); + mongodb.Binary.prototype[util.inspect.custom] = function custom_inspect_binary() { return ``; }; @@ -54,7 +57,7 @@ function decode_json(schema, val) { return val; } if (schema.objectid === true) { - return new mongodb.ObjectId(val); + return new ObjectID(val); } if (schema.date === true) { return new Date(val); @@ -97,7 +100,7 @@ function encode_json(schema, val) { const ops = handle_ops_encoding(schema, val); if (ops) return ops; - if (schema.objectid === true && val instanceof mongodb.ObjectID) { + if (schema.objectid === true && val instanceof ObjectID) { return val.toString(); } @@ -1162,7 +1165,7 @@ class PostgresTable { const new_row = {}; for (const key of Object.keys(row)) { if (key === '_id') { - new_row._id = new mongodb.ObjectID(row[key]); + new_row._id = new ObjectID(row[key]); } else { new_row[key] = parseInt(row[key], 10); } @@ -1585,7 +1588,7 @@ class PostgresClient extends EventEmitter { } generate_id() { - return new mongodb.ObjectId(); + return new ObjectID(null); } collection(name) { @@ -1688,8 +1691,8 @@ class PostgresClient extends EventEmitter { } /** - * make a list of ObjectId unique by indexing their string value - * this is needed since ObjectId is an object so === comparison is not + * make a list of ObjectID unique by indexing their string value + * this is needed since ObjectID is an object so === comparison is not * logically correct for it even for two objects with the same id. */ uniq_ids(docs, doc_path) { @@ -1732,7 +1735,7 @@ class PostgresClient extends EventEmitter { resolve_object_ids_recursive(idmap, item) { _.each(item, (val, key) => { - if (val instanceof mongodb.ObjectId) { + if (val instanceof ObjectID) { if (key !== '_id') { const obj = idmap[val.toHexString()]; if (obj) { @@ -1771,7 +1774,7 @@ class PostgresClient extends EventEmitter { * @returns {nb.ID} */ new_object_id() { - return new mongodb.ObjectId(); + return new ObjectID(null); } /** @@ -1779,20 +1782,20 @@ class PostgresClient extends EventEmitter { * @returns {nb.ID} */ parse_object_id(id_str) { - return new mongodb.ObjectId(String(id_str || undefined)); + return new ObjectID(String(id_str || undefined)); } fix_id_type(doc) { if (_.isArray(doc)) { _.each(doc, d => this.fix_id_type(d)); } else if (doc && doc._id) { - doc._id = new mongodb.ObjectId(doc._id); + doc._id = new ObjectID(doc._id); } return doc; } is_object_id(id) { - return (id instanceof mongodb.ObjectId); + return (id instanceof ObjectID); } // TODO: Figure out error codes diff --git a/src/util/schema_keywords.js b/src/util/schema_keywords.js index 1deb1de3c0..fd3c72882d 100644 --- a/src/util/schema_keywords.js +++ b/src/util/schema_keywords.js @@ -61,26 +61,51 @@ const KEYWORDS = js_utils.deep_freeze({ }, // schema: { objectid: true } will match (new mongodb.ObjectId()) or (new mongodb.ObjectId()).valueOf() + // objectid: { + // keyword: 'objectid', + // // schemaType: 'boolean', + // /** + // * + // * @param {KeywordCxt} cxt + // * + // */ + // code(cxt) { + // const d = cxt.it.data; + // cxt.gen + // .if(CG ` + // typeof ${d} === 'object' && + // ${d} && + // ${d}.constructor && + // ${d}.constructor.name === 'ObjectID' + // `) + // .elseIf(CG ` + // typeof ${d} === 'string' && + // /^[0-9a-fA-F]{24}$/.test(${d}) + // `) + // .else() + // .code(() => cxt.error()) + // .endIf(); + // } + // }, + objectid: { keyword: 'objectid', - // schemaType: 'boolean', /** - * - * @param {KeywordCxt} cxt - * + * Validation logic for the custom `objectid` keyword. + * @param {KeywordCxt} cxt - The validation context. */ code(cxt) { const d = cxt.it.data; cxt.gen .if(CG ` typeof ${d} === 'object' && - ${d} && - ${d}.constructor && + ${d} && + ${d}.constructor && ${d}.constructor.name === 'ObjectID' `) .elseIf(CG ` typeof ${d} === 'string' && - /^[0-9a-fA-F]{24}$/.test(${d}) + require('./objectid').isValid(${d}) `) .else() .code(() => cxt.error())