upgrade mongodb driver

jest.config.js

@@ -57,6 +57,9 @@ module.exports = {
                 '/node_modules/(?!quick-lru)',
             ],
             testTimeout: 8000,
+            setupFilesAfterEnv: [
+                './jest-setup.js',
+            ]
         },
     ],
 };

package.json

@@ -24,7 +24,7 @@
         "clean:modules": "modclean -Pr -n ./modclean.config.js:*",
         "clean:workers": "modclean -Pr -n ./modclean.config.js:* -D workers",
         "clean:packages": "modclean -Pr -n ./modclean.config.js:* -D packages",
-        "test:api:e2e": "./node_modules/jest/bin/jest.js --colors --runInBand --detectOpenHandles -c src/api/e2e/jest.config.js src/api/e2e",
+        "test:api:e2e": "./node_modules/jest/bin/jest.js --colors --runInBand --force-exit --detectOpenHandles -c src/api/e2e/jest.config.js src/api/e2e",
         "test:api:e2e:watch": "./node_modules/jest/bin/jest.js --colors --runInBand --detectOpenHandles -c src/api/e2e/jest.config.js src/api/e2e --watch",
         "test:unit": "./node_modules/jest/bin/jest.js --force-exit --detectOpenHandles --colors",
         "test:unit:watch": "./node_modules/jest/bin/jest.js --watch --colors",
@@ -141,7 +141,7 @@
         "mime": "1.6.0",
         "mime-types": "2.1.35",
         "moment": "2.29.4",
-        "mongodb": "4.17.1",
+        "mongodb": "6.8.0",
         "mongodb-restore": "1.6.2",
         "mui-file-dropzone": "4.0.2",
         "multistream": "^2.1.1",
@@ -236,7 +236,7 @@
     },
     "devDependencies": {
         "@babel/eslint-parser": "^7.23.10",
-        "@shelf/jest-mongodb": "3.0.2",
+        "@shelf/jest-mongodb": "4.3.2",
         "@testing-library/jest-dom": "5.16.5",
         "@testing-library/react": "11.2.7",
         "@types/ejs": "3.1.5",

packages/ezsLodex/jest-setup.js

@@ -0,0 +1 @@
+global.performance = require('perf_hooks').performance;

packages/ezsLodex/package.json

@@ -5,7 +5,7 @@
     "version": "1.16.4",
     "dependencies": {
         "lodash": "4.17.21",
-        "mongodb": "4.17.0",
+        "mongodb": "6.8.0",
         "node-object-hash": "2.3.10",
         "quick-lru": "4.0.1",
         "relaxed-json": "1.0.3",

packages/ezsLodex/src/reduceQuery.js

@@ -55,7 +55,15 @@ export const createFunction = () => async function LodexReduceQuery(data, feed)
     const collName = String('mp_').concat(
         hashCoerce.hash({ reducer, fields }),
     );
-    const options = {
+    const connectionStringURI = this.getParam(
+        'connectionStringURI',
+        data.connectionStringURI || '',
+    );
+    const db = await mongoDatabase(connectionStringURI);
+    const command = {
+        mapReduce: 'publishedDataset',
+        map: map.toString(),
+        reduce: reduce.toString(),
         query: filter,
         finalize,
         out: {
@@ -65,16 +73,13 @@ export const createFunction = () => async function LodexReduceQuery(data, feed)
             fields,
         },
     };
-    const connectionStringURI = this.getParam(
-        'connectionStringURI',
-        data.connectionStringURI || '',
-    );
-    const db = await mongoDatabase(connectionStringURI);
-    const collection = db.collection('publishedDataset');
-
-    const result = await collection.mapReduce(map, reduce, options);
+    const { result: collectionResult, ok } = await db.command(command);
+    if (ok !== 1) {
+        return feed.stop(new Error('MongoDB command return an error'))
+    }
 
-    const total = await result.count();
+    const result = await db.collection(collectionResult);
+    const total = await result.estimatedDocumentCount();
 
     const findFilter = {};
 
@@ -101,6 +106,7 @@ export const createFunction = () => async function LodexReduceQuery(data, feed)
     const cursor = result.find(findFilter);
     const count = await cursor.count();
 
+
     if (total === 0 || count === 0) {
         return feed.send({ total: 0 });
     }
@@ -110,14 +116,15 @@ export const createFunction = () => async function LodexReduceQuery(data, feed)
         path.push('referer');
         value.push(referer);
     }
+
     const stream = cursor
         .sort(sort)
         .skip(skip)
         .limit(limit)
         .stream()
         .on('error', (e) => feed.stop(e))
         .pipe(ezs('assign', { path, value }));
-    await feed.flow(stream);
+    feed.flow(stream);
 };
 
 export default {

packages/ezsLodex/test/mongoQueries.spec.js

@@ -1,3 +1,4 @@
+import {performance, PerformanceObserver} from 'perf_hooks';
 import from from 'from';
 import { MongoClient } from 'mongodb';
 import _ from 'lodash';
@@ -14,22 +15,25 @@ ezs.use(ezsLodex);
 
 describe('mongo queries', () => {
     const connectionStringURI = process.env.MONGO_URL;
+    let client;
     let connection;
     let db;
 
     beforeAll(async () => {
-        connection = await MongoClient.connect(process.env.MONGO_URL, {
-            useNewUrlParser: true,
-            useUnifiedTopology: true,
-        });
-        db = await connection.db();
+        client = new MongoClient(process.env.MONGO_URL);
+        try {
+            await client.connect();
+            db = await client.db();
+        } catch (e) {
+            console.error(`Unable to connect to ${connectionStringURI}` , e);
+        }
     });
 
     afterAll(async () => {
         await Promise.all(
-            Object.keys(handles).map(key => handles[key].close()),
+            Object.keys(handles).map((key) => handles[key].close()),
         );
-        await connection.close();
+        await client.close();
     });
 
     const initDb = (url, data) => {
@@ -40,11 +44,11 @@ describe('mongo queries', () => {
         return Promise.all(requests);
     };
 
-    const drop = () => db.dropDatabase();
+    const drop = async () => await db.dropDatabase();
 
     describe('getCharacteristics', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedCharacteristic));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedCharacteristic));
+        afterEach(async () => await drop());
 
         it('should return characteristics', done => {
             let res = [];
@@ -93,8 +97,8 @@ describe('mongo queries', () => {
     });
 
     describe('getFields', () => {
-        beforeEach(() => initDb(connectionStringURI, field));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, field));
+        afterEach(async () => await drop());
 
         it('should return the fields', done => {
             let res = [];
@@ -137,8 +141,8 @@ describe('mongo queries', () => {
     });
 
     describe('runQuery', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedDataset));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedDataset));
+        afterEach(async () => await drop());
 
         it('should return results', done => {
             let res = [];
@@ -306,8 +310,8 @@ describe('mongo queries', () => {
     });
 
     describe('reduceQuery', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedDataset));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedDataset));
+        afterEach(async () => await drop());
 
         it('should throw when no reducer is given', done => {
             from([
@@ -360,6 +364,8 @@ describe('mongo queries', () => {
                         field: 'publicationDate',
                     }),
                 )
+                .pipe(ezs.catch())
+                .on('error', done)
                 .on('data', data => {
                     res = [...res, data];
                 })
@@ -506,7 +512,7 @@ describe('mongo queries', () => {
                 initDb(connectionStringURI, field),
             ]),
         );
-        afterEach(() => drop());
+        afterEach(async () => await drop());
 
         it('should inject title & summary in each item', done => {
             const res = [];
@@ -555,8 +561,8 @@ describe('mongo queries', () => {
     });
 
     describe('injectDatasetFields', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedCharacteristic));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedCharacteristic));
+        afterEach(async () => await drop());
 
         it('should inject dataset fiels in each item', done => {
             const res = [];
@@ -593,8 +599,8 @@ describe('mongo queries', () => {
     });
 
     describe('labelizeFieldID', () => {
-        beforeEach(() => initDb(connectionStringURI, field));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, field));
+        afterEach(async () => await drop());
 
         const input = [
             {
@@ -668,8 +674,8 @@ describe('mongo queries', () => {
         });
     });
     describe('buildContext', () => {
-        beforeEach(() => initDb(connectionStringURI, field));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, field));
+        afterEach(async () => await drop());
 
         it('with a standard context', done => {
             const res = [];
@@ -745,8 +751,8 @@ describe('mongo queries', () => {
     });
 
     describe('countField', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedDataset));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedDataset));
+        afterEach(async () => await drop());
 
         it('should return results', done => {
             let res = [];
@@ -776,8 +782,8 @@ describe('mongo queries', () => {
     });
 
     describe('aggregateQuery', () => {
-        beforeEach(() => initDb(connectionStringURI, publishedDataset));
-        afterEach(() => drop());
+        beforeEach(async () => await initDb(connectionStringURI, publishedDataset));
+        afterEach(async () => await drop());
 
         it('should return results', done => {
             let res = [];
@@ -898,10 +904,10 @@ describe('mongo queries', () => {
     });
 
     describe('#joinQuery', () => {
-        beforeEach(() =>
-            initDb(connectionStringURI, publishedDatasetWithSubResource),
+        beforeEach(async () =>
+            await initDb(connectionStringURI, publishedDatasetWithSubResource),
         );
-        afterEach(() => drop());
+        afterEach(async () => await drop());
 
         it('should return no results with parameters matchField, matchValue, joinField as empty string', done => {
             const results = [];
@@ -1097,7 +1103,7 @@ describe('mongo queries', () => {
             await initDb(connectionStringURI, precomputedDataset);
         });
 
-        afterEach(() => drop());
+        afterEach(async () => await drop());
 
         it('should return 5 result for segments precomputed', done => {
             const results = [];

src/api/app.js

@@ -159,6 +159,7 @@ app.use(function* (next) {
     try {
         yield next;
     } catch (err) {
+        console.error('Sending error as a body', err);
         this.status = err.status || 500;
         this.body = err.message;
         this.app.emit('error', err, this);

src/api/controller/api/field.js

@@ -20,7 +20,7 @@ import {
 } from '../../../common/scope';
 import { dropJobs } from '../../workers/tools';
 import { ENRICHER } from '../../workers/enricher';
-import { ObjectID } from 'mongodb';
+import { ObjectId } from 'mongodb';
 import generateUid from '../../services/generateUid';
 import { restoreEnrichments } from '../../services/enrichment/enrichment';
 import { restorePrecomputed } from '../../services/precomputed/precomputed';
@@ -35,7 +35,7 @@ export const restoreFields = (fileStream, ctx) => {
             .then((fieldsString) => JSON.parse(fieldsString))
             .then((fields) => {
                 ctx.field
-                    .remove({})
+                    .drop()
                     .then(() =>
                         Promise.all(
                             fields
@@ -70,7 +70,7 @@ export const restoreFields = (fileStream, ctx) => {
     };
 
     return ctx.field
-        .remove({})
+        .drop()
         .then(restoreTask)
         .then(() =>
             Promise.all([
@@ -156,6 +156,7 @@ export const setup = async (ctx, next) => {
     try {
         await next();
     } catch (error) {
+        console.error('Sending error as a body', error);
         ctx.status = 500;
         ctx.body = { error: error.message };
     }
@@ -228,7 +229,7 @@ export const patchSearchableFields = async (ctx) => {
     const fields = ctx.request.body;
 
     try {
-        const ids = fields.map((field) => new ObjectID(field._id));
+        const ids = fields.map((field) => new ObjectId(field._id));
         await ctx.field.updateMany(
             { _id: { $in: ids } },
             { $set: { searchable: true } },
@@ -314,6 +315,7 @@ export const importFields = (asyncBusboyImpl) => async (ctx) => {
         };
         ctx.status = 200;
     } catch (e) {
+        console.error('Sending error as a body', e);
         ctx.status = 500;
         ctx.body = e.message;
     }

src/api/controller/api/publish.spec.js

@@ -12,7 +12,9 @@ jest.mock('../../workers', () => ({
 describe.skip('publish', () => {
     describe('doPublish', () => {
         beforeAll(async () => {
-            await doPublish({});
+            await doPublish({
+                tenant: 'lodex_test',
+            });
         });
         it('should add event to publisher queue', () => {
             expect(workerQueues['lodex_test'].add).toHaveBeenCalledWith({

src/api/controller/api/publishFacets.js

@@ -19,7 +19,7 @@ export default async (ctx, fields, withProgress = false) => {
     jobLogger.info(ctx.job, 'Publishing facets');
 
     const names = fields.map(({ name }) => name);
-    await ctx.publishedFacet.remove({ field: { $in: names } });
+    await ctx.publishedFacet.deleteOne({ field: { $in: names } });
 
     await facetFields
         .reduce(