From 4e023cd90d5c2097d38b2eb737d6862b4d8763f8 Mon Sep 17 00:00:00 2001 From: Lucas De Morais Date: Fri, 13 Dec 2024 17:20:01 +0100 Subject: [PATCH] fix: records not being filtered by schema on live updates --- server/src/__mocks__/collection.js | 4 +- server/src/services/content-types.js | 3 +- server/src/services/live-updates.js | 2 +- server/src/services/orama-manager.js | 90 ++++++++++++++--------- server/src/services/orama-manager.test.js | 72 +++++++++++------- 5 files changed, 105 insertions(+), 66 deletions(-) diff --git a/server/src/__mocks__/collection.js b/server/src/__mocks__/collection.js index 8b09142..3b05b15 100644 --- a/server/src/__mocks__/collection.js +++ b/server/src/__mocks__/collection.js @@ -3,9 +3,9 @@ const mockCollection = { entity: 'testEntity', indexId: 'testIndexId', status: 'outdated', - searchableAttributes: ['name', 'description'], + searchableAttributes: ['title', 'description'], schema: { - name: { type: 'string' }, + title: { type: 'string' }, description: { type: 'string' } }, includedRelations: ['relation1', 'relation2'] diff --git a/server/src/services/content-types.js b/server/src/services/content-types.js index 2b05c8c..73e6e1e 100644 --- a/server/src/services/content-types.js +++ b/server/src/services/content-types.js @@ -93,13 +93,14 @@ module.exports = ({ strapi }) => { })) }, - async getEntries({ contentType, relations = [], schema, offset = 0, limit = 50 }) { + async getEntries({ contentType, relations = [], schema, offset = 0, limit = 50, where }) { const selectedRelations = getSelectedRelations({ schema, relations }) const selectedFields = getSelectedFieldsConfigObj(schema) return await strapi.query(contentType).findMany({ populate: selectedRelations, select: selectedFields, + ...(where ? { where } : {}), limit, offset }) diff --git a/server/src/services/live-updates.js b/server/src/services/live-updates.js index eb72d34..80020c7 100644 --- a/server/src/services/live-updates.js +++ b/server/src/services/live-updates.js @@ -19,7 +19,7 @@ module.exports = ({ strapi }) => { hookManagerService.unregisterHooks(collection) hookManagerService.registerHooks(collection, { async afterCreate(event) { - await handleLiveUpdates(event, 'create') + await handleLiveUpdates(event, 'insert') }, async afterUpdate(event) { await handleLiveUpdates(event, 'update') diff --git a/server/src/services/orama-manager.js b/server/src/services/orama-manager.js index 8c10438..0ec34be 100644 --- a/server/src/services/orama-manager.js +++ b/server/src/services/orama-manager.js @@ -3,6 +3,16 @@ const { CloudManager } = require('@oramacloud/client') const { getSelectedPropsFromObj } = require('../../../utils') +const getAssociatedActionName = (action) => { + const actionMap = { + insert: 'upsert', + update: 'upsert', + delete: 'delete' + } + + return actionMap[action] +} + class OramaManager { constructor({ strapi }) { this.strapi = strapi @@ -13,8 +23,7 @@ class OramaManager { this.oramaCloudManager = new CloudManager({ api_key: this.privateApiKey }) this.DocumentActionsMap = { - create: this.oramaInsert.bind(this), - update: this.oramaUpdate.bind(this), + upsert: this.oramaUpsert.bind(this), delete: this.oramaDelete.bind(this) } } @@ -153,9 +162,11 @@ class OramaManager { }) if (entries.length > 0) { - await this.oramaInsert({ - indexId: collection.indexId, - entries + await this.oramaUpsert({ + collection, + entries, + action: 'insert', + isFromBulk: true }) return await this.bulkInsert(collection, offset + entries.length) @@ -175,45 +186,38 @@ class OramaManager { } /* - * Inserts new documents into the index in the Orama Cloud using the OramaCloud SDK + * Updates documents of the specified index in the Orama Cloud using the OramaCloud SDK * Formats data before insertion using the documentsTransformer function, if provided * @param {string} indexId - Index ID * @param {Array} entries - Array of entries * */ - async oramaInsert({ indexId, entries }) { + async oramaUpsert({ collection: { indexId, entity, schema, includedRelations }, action, entries, isFromBulk }) { + let filteredEntries = entries const index = this.oramaCloudManager.index(indexId) - const transformedData = this.documentsTransformer(indexId, entries) - if (!transformedData) { - this.strapi.log.error(`ERROR: documentsTransformer needs a return value`) - return false + if (!isFromBulk) { + filteredEntries = await this.contentTypesService.getEntries({ + contentType: entity, + relations: includedRelations, + schema: schema, + where: { + id: { + $in: entries.map(({ id }) => id) + } + } + }) } - const result = await index.insert(transformedData) - - this.strapi.log.info(`INSERT: documents with id ${transformedData.map(({ id }) => id)} into index ${indexId}`) - - return result - } - - /* - * Updates documents of the specified index in the Orama Cloud using the OramaCloud SDK - * Formats data before insertion using the documentsTransformer function, if provided - * @param {string} indexId - Index ID - * @param {Array} entries - Array of entries - * */ - async oramaUpdate({ indexId, entries }) { - const index = this.oramaCloudManager.index(indexId) - const transformedData = this.documentsTransformer(indexId, entries) + const transformedData = this.documentsTransformer(indexId, filteredEntries) if (!transformedData) { this.strapi.log.error(`ERROR: documentsTransformer needs a return value`) return false } - const result = await index.update(transformedData) + const result = await index[action](transformedData) - this.strapi.log.info(`UPDATE: document with id ${transformedData.map(({ id }) => id)} into index ${indexId}`) + this.strapi.log.info(`${action.toUpperCase()}: document with id ${transformedData.map(({ id }) => id)} into index ${indexId}`) return result } @@ -223,7 +227,7 @@ class OramaManager { * @param {string} indexId - Index ID * @param {Array} entries - Array of entries * */ - async oramaDelete({ indexId, entries }) { + async oramaDelete({ collection: { indexId }, entries }) { const index = this.oramaCloudManager.index(indexId) const result = await index.delete(entries.map(({ id }) => id)) @@ -239,14 +243,18 @@ class OramaManager { * @param {Object} record - Record object * @param {string} action - Action to perform (insert, update, delete) * */ - async handleDocument({ indexId, record, action }) { - if (!action || !record || !this.DocumentActionsMap[action]) { + async handleDocument({ collection, record, action }) { + const associatedActionName = getAssociatedActionName(action) + + if (!action || !record || !this.DocumentActionsMap[associatedActionName]) { + this.strapi.log.warn(`Action ${action} not found. Skipping...`) + return false } const { createdBy, updatedBy, ...rest } = record - return await this.DocumentActionsMap[action]({ indexId, entries: [{ ...rest, id: rest.id.toString() }] }) + return await this.DocumentActionsMap[associatedActionName]({ collection, action, entries: [{ ...rest, id: rest.id.toString() }] }) } /* @@ -302,8 +310,22 @@ class OramaManager { return } + const customSchema = this.collectionSettings?.[collection.indexId]?.schema + + const oramaSchema = + customSchema ?? + getSelectedPropsFromObj({ + props: collection.searchableAttributes, + obj: collection.schema + }) + await this.updatingStarted(collection) + await this.oramaUpdateSchema({ + indexId: collection.indexId, + schema: oramaSchema + }) + await this.oramaDeployIndex(collection) await this.updatingCompleted(collection) @@ -328,7 +350,7 @@ class OramaManager { await this.updatingStarted(collection) const handleDocumentResult = await this.handleDocument({ - indexId: collection.indexId, + collection, record, action }) diff --git a/server/src/services/orama-manager.test.js b/server/src/services/orama-manager.test.js index e3f6bd9..5a0d3ba 100644 --- a/server/src/services/orama-manager.test.js +++ b/server/src/services/orama-manager.test.js @@ -20,6 +20,7 @@ const strapi = { log: { error: jest.fn(), debug: jest.fn(), + warn: jest.fn(), info: jest.fn() } } @@ -242,15 +243,17 @@ describe('OramaManager', () => { describe('bulkInsert', () => { it('should insert entries', async () => { - const oramaInsertSpy = jest.spyOn(oramaManager, 'oramaInsert').mockResolvedValue() + const oramaUpsertSpy = jest.spyOn(oramaManager, 'oramaUpsert').mockResolvedValue() const bulkInsertSpy = jest.spyOn(oramaManager, 'bulkInsert') await oramaManager.bulkInsert(mockCollection) expect(bulkInsertSpy).toHaveBeenNthCalledWith(1, mockCollection) - expect(oramaInsertSpy).toHaveBeenCalledWith({ - indexId: mockCollection.indexId, - entries: [{ id: 1, title: 'Test Entry' }] + expect(oramaUpsertSpy).toHaveBeenCalledWith({ + collection: mockCollection, + entries: [{ id: 1, title: 'Test Entry' }], + action: 'insert', + isFromBulk: true }) expect(bulkInsertSpy).toHaveBeenLastCalledWith(mockCollection, 1) }) @@ -266,14 +269,16 @@ describe('OramaManager', () => { }) }) - describe('oramaInsert', () => { + describe('oramaUpsert - insert', () => { it('should insert entries', async () => { const documentsTransformerSpy = jest.spyOn(oramaManager, 'documentsTransformer') const { insert } = new CloudManager({ strapi }).index() - await oramaManager.oramaInsert({ - indexId: mockCollection.indexId, - entries: [{ id: 1, title: 'Test Entry' }] + await oramaManager.oramaUpsert({ + collection: mockCollection, + entries: [{ id: 1, title: 'Test Entry' }], + action: 'insert', + isFromBulk: true }) expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) @@ -296,9 +301,11 @@ describe('OramaManager', () => { const documentsTransformerSpy = jest.spyOn(oramaManager, 'documentsTransformer') const { insert } = new CloudManager({ strapi }).index() - await oramaManager.oramaInsert({ - indexId: mockCollection.indexId, - entries: [{ id: 1, title: 'Test Entry' }] + await oramaManager.oramaUpsert({ + collection: mockCollection, + entries: [{ id: 1, title: 'Test Entry' }], + action: 'insert', + isFromBulk: true }) expect(documentsTransformerSpy).toHaveBeenCalledWith(mockCollection.indexId, [{ id: 1, title: 'Test Entry' }]) @@ -314,13 +321,18 @@ describe('OramaManager', () => { }) }) - describe('oramaUpdate', () => { + describe('oramaUpsert - update', () => { + beforeEach(() => { + contentTypesService.getEntries.mockResolvedValue([{ id: 1, title: 'Test Entry' }]) + }) + it('should update entries', async () => { const { update } = new CloudManager({ strapi }).index() - await oramaManager.oramaUpdate({ - indexId: mockCollection.indexId, - entries: [{ id: 1, title: 'Test Entry' }] + await oramaManager.oramaUpsert({ + collection: mockCollection, + entries: [{ id: 1, title: 'Test Entry' }], + action: 'update' }) expect(update).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) @@ -328,7 +340,11 @@ describe('OramaManager', () => { }) describe('handleDocument', () => { - it('should return if action is not found', async () => { + beforeEach(() => { + contentTypesService.getEntries.mockResolvedValue([{ id: 1, title: 'Test Entry' }]) + }) + + it('should return false if action is not found', async () => { const { insert } = new CloudManager({ strapi }).index() await oramaManager.handleDocument({ @@ -358,36 +374,36 @@ describe('OramaManager', () => { const { insert } = new CloudManager({ strapi }).index() await oramaManager.handleDocument({ - indexId: mockCollection.indexId, + collection: mockCollection, record: mockedTestRecord, - action: 'create' + action: 'insert', }) - expect(insert).toHaveBeenCalledWith([{ id: '1', title: 'Test Entry' }]) + expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) }) it('should return if action is update', async () => { const { update } = new CloudManager({ strapi }).index() await oramaManager.handleDocument({ - indexId: mockCollection.indexId, + collection: mockCollection, record: mockedTestRecord, action: 'update' }) - expect(update).toHaveBeenCalledWith([{ id: '1', title: 'Test Entry' }]) + expect(update).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) }) it('should return if action is delete', async () => { const { delete: deleteFn } = new CloudManager({ strapi }).index() await oramaManager.handleDocument({ - indexId: mockCollection.indexId, + collection: mockCollection, record: mockedTestRecord, action: 'delete' }) - expect(deleteFn).toHaveBeenCalledWith(['1']) + expect(deleteFn).toHaveBeenCalledWith(["1"]) }) }) @@ -465,14 +481,14 @@ describe('OramaManager', () => { describe('processLiveUpdate', () => { it('should process live update', async () => { - await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'create') + await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'insert') expect(oramaManager.validate).toHaveBeenCalledWith(mockCollection) expect(updatingStartedSpy).toHaveBeenCalledWith(mockCollection) expect(handleDocumentSpy).toHaveBeenCalledWith({ - indexId: mockCollection.indexId, + collection: mockCollection, record: mockedTestRecord, - action: 'create' + action: 'insert' }) expect(setOutdatedSpy).toHaveBeenCalledWith(mockCollection) }) @@ -480,7 +496,7 @@ describe('OramaManager', () => { it('should not process live update if collection is not valid', async () => { collectionService.findOne.mockReturnValueOnce(mockNotValidCollection) - await oramaManager.processLiveUpdate(mockNotValidCollection, mockedTestRecord, 'create') + await oramaManager.processLiveUpdate(mockNotValidCollection, mockedTestRecord, 'insert') expect(oramaManager.validate).toHaveBeenCalledWith(mockNotValidCollection) expect(updatingStartedSpy).not.toHaveBeenCalled() @@ -494,7 +510,7 @@ describe('OramaManager', () => { //Force handleDocument to reject handleDocumentSpy.mockResolvedValueOnce(false) - await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'create') + await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'insert') expect(oramaManager.validate).toHaveBeenCalledWith(mockCollection) expect(updatingStartedSpy).toHaveBeenCalled()