Skip to content

Commit

Permalink
Merge pull request #8508 from dannyzaken/danny-postgres-pools
Browse files Browse the repository at this point in the history
  • Loading branch information
dannyzaken authored Dec 2, 2024
2 parents 9bce005 + 7ddd87d commit 3aabb04
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 83 deletions.
7 changes: 4 additions & 3 deletions config.js
Original file line number Diff line number Diff line change
Expand Up @@ -199,15 +199,15 @@ config.S3_RESTORE_REQUEST_MAX_DAYS_BEHAVIOUR = 'TRUNCATE';
/**
* S3_MAX_KEY_LENGTH controls the maximum key length that will be accepted
* by NooBaa endpoints.
*
*
* This value is 1024 bytes for S3 but the default is `Infinity`
*/
config.S3_MAX_KEY_LENGTH = Infinity;

/**
* S3_MAX_BUCKET_NAME_LENGTH controls the maximum bucket name length that
* will be accepted by NooBaa endpoints.
*
*
* This value is 63 bytes for S3 but the default is `Infinity`
*/
config.S3_MAX_BUCKET_NAME_LENGTH = Infinity;
Expand All @@ -229,7 +229,8 @@ config.ROOT_KEY_MOUNT = '/etc/noobaa-server/root_keys';

config.DB_TYPE = /** @type {nb.DBType} */ (process.env.DB_TYPE || 'postgres');

config.POSTGRES_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 80 : 10;
config.POSTGRES_DEFAULT_MAX_CLIENTS = 10;
config.POSTGRES_MD_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 70 : 10;

///////////////////
// SYSTEM CONFIG //
Expand Down
2 changes: 1 addition & 1 deletion src/deploy/NVA_build/standalone_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const config = exports;
config.DEFAULT_POOL_TYPE = 'HOSTS';
config.AGENT_RPC_PORT = '9999';
config.AGENT_RPC_PROTOCOL = 'tcp';
config.POSTGRES_MAX_CLIENTS = 10;
config.POSTGRES_DEFAULT_MAX_CLIENTS = 10;
EOF

# setup_env is not needed when running inside a container because the container
Expand Down
50 changes: 28 additions & 22 deletions src/server/object_services/md_store.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,33 +30,43 @@ const config = require('../../../config');
class MDStore {

constructor(test_suffix = '') {


const postgres_pool = 'md';

this._objects = db_client.instance().define_collection({
name: 'objectmds' + test_suffix,
schema: object_md_schema,
db_indexes: object_md_indexes,
postgres_pool,
});
this._multiparts = db_client.instance().define_collection({
name: 'objectmultiparts' + test_suffix,
schema: object_multipart_schema,
db_indexes: object_multipart_indexes,
postgres_pool,
});
this._parts = db_client.instance().define_collection({
name: 'objectparts' + test_suffix,
schema: object_part_schema,
db_indexes: object_part_indexes,
postgres_pool,
});
this._chunks = db_client.instance().define_collection({
name: 'datachunks' + test_suffix,
schema: data_chunk_schema,
db_indexes: data_chunk_indexes,
postgres_pool,
});
this._blocks = db_client.instance().define_collection({
name: 'datablocks' + test_suffix,
schema: data_block_schema,
db_indexes: data_block_indexes,
postgres_pool,
});
this._sequences = db_client.instance().define_sequence({
name: 'mdsequences' + test_suffix,
postgres_pool,
});
}

Expand Down Expand Up @@ -248,19 +258,16 @@ class MDStore {
async remove_objects_and_unset_latest(objs) {
if (!objs || !objs.length) return;

await this._objects.updateMany(
{
_id: {
$in: objs.map(obj => obj._id),
}
},
{
$set: {
deleted: new Date(),
version_past: true,
},
await this._objects.updateMany({
_id: {
$in: objs.map(obj => obj._id),
}
);
}, {
$set: {
deleted: new Date(),
version_past: true,
},
});
}

// 2, 3, 4
Expand Down Expand Up @@ -1362,7 +1369,7 @@ class MDStore {
}

/**
*
*
* @param {{
* tier: nb.ID,
* limit: number,
Expand All @@ -1387,14 +1394,14 @@ class MDStore {
}

return this._chunks
.find(selectors, {
projection: { _id: 1 },
hint: "tiering_index",
sort,
limit,
})
.find(selectors, {
projection: { _id: 1 },
hint: "tiering_index",
sort,
limit,
})

.then(chunks => db_client.instance().uniq_ids(chunks, "_id"));
.then(chunks => db_client.instance().uniq_ids(chunks, "_id"));
}


Expand Down Expand Up @@ -1775,8 +1782,7 @@ class MDStore {
find_deleted_blocks(max_delete_time, limit) {
const query = {
deleted: {
$lt: new Date(max_delete_time),
$exists: true // Force index usage
$lt: new Date(max_delete_time)
},
};
return this._blocks.find(query, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ export JWT_SECRET=123456789
export NOOBAA_ROOT_SECRET='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='
export LOCAL_MD_SERVER=true

#The default max connections for postgres is 100. limit max clients to 10 per pool (per process).
export CONFIG_JS_POSTGRES_MD_MAX_CLIENTS=10
export CONFIG_JS_POSTGRES_DEFAULT_MAX_CLIENTS=10

export POSTGRES_HOST=${POSTGRES_HOST:-localhost}
export MGMT_ADDR=wss://${NOOBAA_MGMT_SERVICE_HOST:-localhost}:${NOOBAA_MGMT_SERVICE_PORT:-5443}
export BG_ADDR=wss://localhost:5445
Expand Down
Loading

0 comments on commit 3aabb04

Please sign in to comment.