diff --git a/packages/shared/akamai-edgeworker-sdk/src/featureStore/index.ts b/packages/shared/akamai-edgeworker-sdk/src/featureStore/index.ts index 18960e1f7..cf968f391 100644 --- a/packages/shared/akamai-edgeworker-sdk/src/featureStore/index.ts +++ b/packages/shared/akamai-edgeworker-sdk/src/featureStore/index.ts @@ -107,6 +107,15 @@ export class EdgeFeatureStore implements LDFeatureStore { } init(allData: LDFeatureStoreDataStorage, callback: () => void): void { + this.applyChanges(true, allData, undefined, callback); + } + + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void { callback(); } diff --git a/packages/shared/sdk-server-edge/src/api/EdgeFeatureStore.ts b/packages/shared/sdk-server-edge/src/api/EdgeFeatureStore.ts index 039ee4a30..55cedb31f 100644 --- a/packages/shared/sdk-server-edge/src/api/EdgeFeatureStore.ts +++ b/packages/shared/sdk-server-edge/src/api/EdgeFeatureStore.ts @@ -100,6 +100,15 @@ export class EdgeFeatureStore implements LDFeatureStore { } init(allData: LDFeatureStoreDataStorage, callback: () => void): void { + this.applyChanges(true, allData, undefined, callback); + } + + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void { callback(); } diff --git a/packages/shared/sdk-server-edge/src/utils/mockFeatureStore.ts b/packages/shared/sdk-server-edge/src/utils/mockFeatureStore.ts index 037bed69e..cd79ce962 100644 --- a/packages/shared/sdk-server-edge/src/utils/mockFeatureStore.ts +++ b/packages/shared/sdk-server-edge/src/utils/mockFeatureStore.ts @@ -6,6 +6,7 @@ const mockFeatureStore: LDFeatureStore = { init: jest.fn(), initialized: jest.fn(), upsert: jest.fn(), + applyChanges: jest.fn(), get: jest.fn(), delete: jest.fn(), }; diff --git a/packages/shared/sdk-server/__tests__/data_sources/createPayloadListenersFDv2.test.ts b/packages/shared/sdk-server/__tests__/data_sources/createPayloadListenersFDv2.test.ts index 967eb658f..e5129a9b5 100644 --- a/packages/shared/sdk-server/__tests__/data_sources/createPayloadListenersFDv2.test.ts +++ b/packages/shared/sdk-server/__tests__/data_sources/createPayloadListenersFDv2.test.ts @@ -88,6 +88,7 @@ describe('createPayloadListenerFDv2', () => { dataSourceUpdates = { init: jest.fn(), upsert: jest.fn(), + applyChanges: jest.fn(), }; basisRecieved = jest.fn(); }); diff --git a/packages/shared/sdk-server/__tests__/data_sources/createStreamListeners.test.ts b/packages/shared/sdk-server/__tests__/data_sources/createStreamListeners.test.ts index 3237b9417..b3aeee0ff 100644 --- a/packages/shared/sdk-server/__tests__/data_sources/createStreamListeners.test.ts +++ b/packages/shared/sdk-server/__tests__/data_sources/createStreamListeners.test.ts @@ -52,6 +52,7 @@ describe('createStreamListeners', () => { dataSourceUpdates = { init: jest.fn(), upsert: jest.fn(), + applyChanges: jest.fn(), }; onPutCompleteHandler = jest.fn(); onPatchCompleteHandler = jest.fn(); diff --git a/packages/shared/sdk-server/src/api/subsystems/LDDataSourceUpdates.ts b/packages/shared/sdk-server/src/api/subsystems/LDDataSourceUpdates.ts index 8b3badd38..53326e309 100644 --- a/packages/shared/sdk-server/src/api/subsystems/LDDataSourceUpdates.ts +++ b/packages/shared/sdk-server/src/api/subsystems/LDDataSourceUpdates.ts @@ -40,4 +40,21 @@ export interface LDDataSourceUpdates { * Will be called after the upsert operation is complete. */ upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void; + + /** + * @param basis If true, completely overwrites the current contents of the data store + * with the provided data. If false, upserts the items in the provided data. Upserts + * are made only if provided items have newer versions than existing items. + * @param data An object in which each key is the "namespace" of a collection (e.g. `"features"`) and + * the value is an object that maps keys to entities. The actual type of this parameter is + * `interfaces.FullDataSet`. + * @param selector TODO + * @param callback Will be called after the changes are applied. + */ + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String, + callback: () => void, + ): void; } diff --git a/packages/shared/sdk-server/src/api/subsystems/LDFeatureStore.ts b/packages/shared/sdk-server/src/api/subsystems/LDFeatureStore.ts index 9bdfb9430..39ce883c3 100644 --- a/packages/shared/sdk-server/src/api/subsystems/LDFeatureStore.ts +++ b/packages/shared/sdk-server/src/api/subsystems/LDFeatureStore.ts @@ -137,6 +137,23 @@ export interface LDFeatureStore { */ upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void; + /** + * @param basis If true, completely overwrites the current contents of the data store + * with the provided data. If false, upserts the items in the provided data. Upserts + * are made only if provided items have newer versions than existing items. + * @param data An object in which each key is the "namespace" of a collection (e.g. `"features"`) and + * the value is an object that maps keys to entities. The actual type of this parameter is + * `interfaces.FullDataSet`. + * @param selector TODO + * @param callback Will be called after the changes are applied. + */ + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void; + /** * Tests whether the store is initialized. * diff --git a/packages/shared/sdk-server/src/data_sources/DataSourceUpdates.ts b/packages/shared/sdk-server/src/data_sources/DataSourceUpdates.ts index ac6e3820d..9de85c737 100644 --- a/packages/shared/sdk-server/src/data_sources/DataSourceUpdates.ts +++ b/packages/shared/sdk-server/src/data_sources/DataSourceUpdates.ts @@ -67,14 +67,38 @@ export default class DataSourceUpdates implements LDDataSourceUpdates { ) {} init(allData: LDFeatureStoreDataStorage, callback: () => void): void { + this.applyChanges(true, allData, undefined, callback); // basis is true for init. selector is undefined for FDv1 init + } + + upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void { + this.applyChanges( + false, // basis is false for upserts + { + [kind.namespace]: { + [data.key]: data, + }, + }, + undefined, // selector is undefined for FDv1 upsert + callback, + ); + } + + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void { const checkForChanges = this._hasEventListeners(); - const doInit = (oldData?: LDFeatureStoreDataStorage) => { - this._featureStore.init(allData, () => { + const doApplyChanges = (oldData: LDFeatureStoreDataStorage) => { + this._featureStore.applyChanges(basis, data, selector, () => { // Defer change events so they execute after the callback. Promise.resolve().then(() => { - this._dependencyTracker.reset(); + if (basis) { + this._dependencyTracker.reset(); + } - Object.entries(allData).forEach(([namespace, items]) => { + Object.entries(data).forEach(([namespace, items]) => { Object.keys(items || {}).forEach((key) => { const item = items[key]; this._dependencyTracker.updateDependenciesFrom( @@ -87,11 +111,18 @@ export default class DataSourceUpdates implements LDDataSourceUpdates { if (checkForChanges) { const updatedItems = new NamespacedDataSet(); - Object.keys(allData).forEach((namespace) => { - const oldDataForKind = oldData?.[namespace] || {}; - const newDataForKind = allData[namespace]; - const mergedData = { ...oldDataForKind, ...newDataForKind }; - Object.keys(mergedData).forEach((key) => { + Object.keys(data).forEach((namespace) => { + const oldDataForKind = oldData[namespace]; + const newDataForKind = data[namespace]; + let iterateData; + if (basis) { + // for basis, need to iterate on all keys + iterateData = { ...oldDataForKind, ...newDataForKind }; + } else { + // for non basis, only need to iterate on keys in incoming data + iterateData = { ...newDataForKind }; + } + Object.keys(iterateData).forEach((key) => { this.addIfModified( namespace, key, @@ -101,6 +132,7 @@ export default class DataSourceUpdates implements LDDataSourceUpdates { ); }); }); + this.sendChangeEvents(updatedItems); } }); @@ -108,48 +140,20 @@ export default class DataSourceUpdates implements LDDataSourceUpdates { }); }; + let oldData = {}; if (checkForChanges) { + // record old data before making changes to use for change calculations this._featureStore.all(VersionedDataKinds.Features, (oldFlags) => { this._featureStore.all(VersionedDataKinds.Segments, (oldSegments) => { - const oldData = { + oldData = { [VersionedDataKinds.Features.namespace]: oldFlags, [VersionedDataKinds.Segments.namespace]: oldSegments, }; - doInit(oldData); }); }); - } else { - doInit(); } - } - upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void { - const { key } = data; - const checkForChanges = this._hasEventListeners(); - const doUpsert = (oldItem?: LDFeatureStoreItem | null) => { - this._featureStore.upsert(kind, data, () => { - // Defer change events so they execute after the callback. - Promise.resolve().then(() => { - this._dependencyTracker.updateDependenciesFrom( - kind.namespace, - key, - computeDependencies(kind.namespace, data), - ); - if (checkForChanges) { - const updatedItems = new NamespacedDataSet(); - this.addIfModified(kind.namespace, key, oldItem, data, updatedItems); - this.sendChangeEvents(updatedItems); - } - }); - - callback?.(); - }); - }; - if (checkForChanges) { - this._featureStore.get(kind, key, doUpsert); - } else { - doUpsert(); - } + doApplyChanges(oldData); } addIfModified( diff --git a/packages/shared/sdk-server/src/store/InMemoryFeatureStore.ts b/packages/shared/sdk-server/src/store/InMemoryFeatureStore.ts index 61814f2aa..0df56210b 100644 --- a/packages/shared/sdk-server/src/store/InMemoryFeatureStore.ts +++ b/packages/shared/sdk-server/src/store/InMemoryFeatureStore.ts @@ -12,22 +12,6 @@ export default class InMemoryFeatureStore implements LDFeatureStore { private _initCalled = false; - private _addItem(kind: DataKind, key: string, item: LDFeatureStoreItem) { - let items = this._allData[kind.namespace]; - if (!items) { - items = {}; - this._allData[kind.namespace] = items; - } - if (Object.hasOwnProperty.call(items, key)) { - const old = items[key]; - if (!old || old.version < item.version) { - items[key] = item; - } - } else { - items[key] = item; - } - } - get(kind: DataKind, key: string, callback: (res: LDFeatureStoreItem | null) => void): void { const items = this._allData[kind.namespace]; if (items) { @@ -53,19 +37,66 @@ export default class InMemoryFeatureStore implements LDFeatureStore { } init(allData: LDFeatureStoreDataStorage, callback: () => void): void { - this._initCalled = true; - this._allData = allData as LDFeatureStoreDataStorage; - callback?.(); + this.applyChanges(true, allData, undefined, callback); } delete(kind: DataKind, key: string, version: number, callback: () => void): void { - const deletedItem = { version, deleted: true }; - this._addItem(kind, key, deletedItem); - callback?.(); + const item: LDKeyedFeatureStoreItem = { key, version, deleted: true }; + this.applyChanges( + false, + { + [kind.namespace]: { + [key]: item, + }, + }, + undefined, + callback, + ); } upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void { - this._addItem(kind, data.key, data); + this.applyChanges( + false, + { + [kind.namespace]: { + [data.key]: data, + }, + }, + undefined, + callback, + ); + } + + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, // TODO handle selector + callback: () => void, + ): void { + if (basis) { + this._initCalled = true; + this._allData = data; + } else { + Object.entries(data).forEach(([namespace, items]) => { + Object.keys(items || {}).forEach((key) => { + let existingItems = this._allData[namespace]; + if (!existingItems) { + existingItems = {}; + this._allData[namespace] = existingItems; + } + const item = items[key]; + if (Object.hasOwnProperty.call(existingItems, key)) { + const old = existingItems[key]; + if (!old || old.version < item.version) { + existingItems[key] = item; + } + } else { + existingItems[key] = item; + } + }); + }); + } + callback?.(); } diff --git a/packages/shared/sdk-server/src/store/PersistentDataStoreWrapper.ts b/packages/shared/sdk-server/src/store/PersistentDataStoreWrapper.ts index c682e4234..cdc814914 100644 --- a/packages/shared/sdk-server/src/store/PersistentDataStoreWrapper.ts +++ b/packages/shared/sdk-server/src/store/PersistentDataStoreWrapper.ts @@ -252,6 +252,27 @@ export default class PersistentDataStoreWrapper implements LDFeatureStore { this.upsert(kind, { key, version, deleted: true }, callback); } + applyChanges( + _basis: boolean, + _data: LDFeatureStoreDataStorage, + _selector: String | undefined, + _callback: () => void, + ): void { + // TODO: SDK-1029 - Transactional persistent store - update this to not iterate over items and instead send data to underlying PersistentDataStore + // no need for queue at the moment as init and upsert handle that, but as part of SDK-1029, queue may be needed + if (_basis) { + this.init(_data, _callback); + } else { + Object.entries(_data).forEach(([namespace, items]) => { + Object.keys(items || {}).forEach((key) => { + const item = items[key]; + this.upsert({ namespace }, { key, ...item }, () => {}); + }); + }); + _callback(); + } + } + close(): void { this._itemCache?.close(); this._allItemsCache?.close(); diff --git a/packages/store/node-server-sdk-dynamodb/src/DynamoDBFeatureStore.ts b/packages/store/node-server-sdk-dynamodb/src/DynamoDBFeatureStore.ts index 100d7b92d..ec248ece7 100644 --- a/packages/store/node-server-sdk-dynamodb/src/DynamoDBFeatureStore.ts +++ b/packages/store/node-server-sdk-dynamodb/src/DynamoDBFeatureStore.ts @@ -51,6 +51,15 @@ export default class DynamoDBFeatureStore implements LDFeatureStore { this._wrapper.upsert(kind, data, callback); } + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void { + this._wrapper.applyChanges(basis, data, selector, callback); + } + initialized(callback: (isInitialized: boolean) => void): void { this._wrapper.initialized(callback); } diff --git a/packages/store/node-server-sdk-redis/src/RedisFeatureStore.ts b/packages/store/node-server-sdk-redis/src/RedisFeatureStore.ts index c01b5d752..be4efd755 100644 --- a/packages/store/node-server-sdk-redis/src/RedisFeatureStore.ts +++ b/packages/store/node-server-sdk-redis/src/RedisFeatureStore.ts @@ -51,6 +51,15 @@ export default class RedisFeatureStore implements LDFeatureStore { this._wrapper.upsert(kind, data, callback); } + applyChanges( + basis: boolean, + data: LDFeatureStoreDataStorage, + selector: String | undefined, + callback: () => void, + ): void { + this._wrapper.applyChanges(basis, data, selector, callback); + } + initialized(callback: (isInitialized: boolean) => void): void { this._wrapper.initialized(callback); }