Configurable free space loading limits

dependencies/lmdb/libraries/liblmdb/lmdb.h

@@ -1068,8 +1068,9 @@ int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 //<lmdb-js>
 typedef int (MDB_check_fd)(const mdb_filehandle_t fd, MDB_env* env);
 typedef void (MDB_txn_visible)(const void* ctx);
-int  mdb_env_set_callback(MDB_env *env, MDB_check_fd *func);
-int  mdb_txn_set_callback(MDB_txn *txn, MDB_txn_visible *func, void* ctx);
+int mdb_env_set_callback(MDB_env *env, MDB_check_fd *func);
+int mdb_txn_set_callback(MDB_txn *txn, MDB_txn_visible *func, void* ctx);
+int	mdb_env_set_freespace_options(MDB_env *env, unsigned int max_to_load, unsigned int max_to_retain);
 //</lmdb-js>
 
 #if MDB_RPAGE_CACHE

dependencies/lmdb/libraries/liblmdb/mdb.c

@@ -1648,6 +1648,8 @@ struct MDB_env {
 #	define		me_pglast	me_pgstate.mf_pglast
 #	define		me_pghead	me_pgstate.mf_pghead
 #	define		me_freelist_position	me_pgstate.mf_position
+	unsigned int me_maxfreepgs_to_load; /**< max freelist entries to load into memory */
+	unsigned int me_maxfreepgs_to_retain; /**< max freelist entries to load into memory */
 	MDB_page	*me_dpages;		/**< list of malloc'd blocks for re-use */
 	/** IDL of pages that became unused in a write txn */
 	MDB_IDL		me_free_pgs;
@@ -2826,8 +2828,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		env->me_freelist_position = i;
 		i = 0;
 
-		if (mop_len > 10000) {
-			//fprintf(stderr, "Too many entries %u, looking for %u, best fit %u, not loading anymore\n", mop_len, num, best_fit_size);
+		if (mop_len > env->me_maxfreepgs_to_load) {
+			fprintf(stderr, "Too many entries %u, looking for %u, best fit %u, not loading anymore\n", mop_len, num, best_fit_size);
 			goto continue_best_fit;
 		}
 
@@ -3882,8 +3884,8 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			mdb_midl_shrink(&txn->mt_free_pgs);
 			env->me_free_pgs = txn->mt_free_pgs;
 			/* me_pgstate: */
-			if (env->me_pghead && env->me_pghead[0] > 20000) {
-				//fprintf(stderr, "Free list too large %u, dumping from memory\n", env->me_pghead[0]);
+			if (env->me_pghead && env->me_pghead[0] > env->me_maxfreepgs_to_retain) {
+				fprintf(stderr, "Free list too large %u, dumping from memory\n", env->me_pghead[0]);
 				// if it is too large, reset it
 				env->me_pghead = NULL;
 				env->me_pglast = 0;
@@ -5204,6 +5206,8 @@ mdb_env_create(MDB_env **env)
 
 	e->me_maxreaders = DEFAULT_READERS;
 	e->me_maxdbs = e->me_numdbs = CORE_DBS;
+	e->me_maxfreepgs_to_load = 20000;
+	e->me_maxfreepgs_to_retain = 10000;
 	e->me_fd = INVALID_HANDLE_VALUE;
 	e->me_lfd = INVALID_HANDLE_VALUE;
 	e->me_mfd = INVALID_HANDLE_VALUE;
@@ -11995,6 +11999,16 @@ mdb_env_set_callback(MDB_env *env, MDB_check_fd *func)
 	return MDB_SUCCESS;
 }
 
+int ESECT
+mdb_env_set_freespace_options(MDB_env *env, unsigned int max_to_load, unsigned int max_to_retain)
+{
+	if (!env)
+		return EINVAL;
+	env->me_maxfreepgs_to_load = max_to_load;
+	env->me_maxfreepgs_to_retain = max_to_retain;
+	return MDB_SUCCESS;
+}
+
 #if MDB_RPAGE_CACHE
 int ESECT
 mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size)

src/env.cpp

@@ -276,6 +276,14 @@ Napi::Value EnvWrap::open(const CallbackInfo& info) {
 	option = options.Get("maxReaders");
 	if (option.IsNumber())
 		maxReaders = option.As<Number>();
+	int maxFreeSpaceToLoad = 20000;
+	option = options.Get("maxFreeSpaceToLoad");
+	if (option.IsNumber())
+		maxFreeSpaceToLoad = option.As<Number>();
+	int maxFreeSpaceToRetain = 10000;
+	option = options.Get("maxFreeSpaceToRetain");
+	if (option.IsNumber())
+		maxFreeSpaceToRetain = option.As<Number>();
 
 	Napi::Value encryptionKey = options.Get("encryptionKey");
 	std::string encryptKey;
@@ -290,15 +298,15 @@ Napi::Value EnvWrap::open(const CallbackInfo& info) {
 	}
 
 	napiEnv = info.Env();
-	rc = openEnv(flags, jsFlags, (const char*)pathString.c_str(), (char*) keyBuffer, compression, maxDbs, maxReaders, mapSize, pageSize, encryptKey.empty() ? nullptr : (char*)encryptKey.c_str());
+	rc = openEnv(flags, jsFlags, (const char*)pathString.c_str(), (char*) keyBuffer, compression, maxDbs, maxReaders, mapSize, pageSize, maxFreeSpaceToLoad, maxFreeSpaceToRetain, encryptKey.empty() ? nullptr : (char*)encryptKey.c_str());
 	//delete[] pathBytes;
 	if (rc != 0)
 		return throwLmdbError(info.Env(), rc);
 	napi_add_env_cleanup_hook(napiEnv, cleanup, this);
 	return info.Env().Undefined();
 }
 int EnvWrap::openEnv(int flags, int jsFlags, const char* path, char* keyBuffer, Compression* compression, int maxDbs,
-		int maxReaders, mdb_size_t mapSize, int pageSize, char* encryptionKey) {
+		int maxReaders, mdb_size_t mapSize, int pageSize, unsigned int max_free_to_load, unsigned int max_free_to_retain, char* encryptionKey) {
 	this->keyBuffer = keyBuffer;
 	this->compression = compression;
 	this->jsFlags = jsFlags;
@@ -316,6 +324,9 @@ int EnvWrap::openEnv(int flags, int jsFlags, const char* path, char* keyBuffer,
 	if (pageSize)
 	   rc = mdb_env_set_pagesize(env, pageSize);
 	if (rc) goto fail;
+	if (max_free_to_load)
+	   rc = mdb_env_set_freespace_options(env, max_free_to_load, max_free_to_retain);
+	if (rc) goto fail;
 	#endif
 	if ((size_t) encryptionKey > 100) {
 		MDB_val enckey;

src/lmdb-js.h

@@ -350,7 +350,7 @@ class EnvWrap : public ObjectWrap<EnvWrap> {
 	static void setupExports(Napi::Env env, Object exports);
 	void closeEnv(bool hasLock = false);
 	int openEnv(int flags, int jsFlags, const char* path, char* keyBuffer, Compression* compression, int maxDbs,
-		int maxReaders, mdb_size_t mapSize, int pageSize, char* encryptionKey);
+		int maxReaders, mdb_size_t mapSize, int pageSize, unsigned int max_free_to_load, unsigned int max_free_to_retain, char* encryptionKey);
 
 	/*
 		Gets statistics about the database environment.

test/index.test.js

@@ -68,6 +68,8 @@ describe('lmdb-js', function () {
 			noMemInit: true,
 			trackMetrics: true,
 			pageSize: 0x2000,
+			maxFreeSpaceToLoad: 400,
+			maxFreeSpaceToRetain: 100,
 		}),
 	);
 	describe(