chore: preparation work for FDv2 feature store and persistence support

packages/shared/akamai-edgeworker-sdk/src/featureStore/index.ts

@@ -107,6 +107,15 @@ export class EdgeFeatureStore implements LDFeatureStore {
   }
 
   init(allData: LDFeatureStoreDataStorage, callback: () => void): void {
+    this.applyChanges(true, allData, undefined, callback);
+  }
+
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void {
     callback();
   }
 

packages/shared/sdk-server-edge/src/api/EdgeFeatureStore.ts

@@ -100,6 +100,15 @@ export class EdgeFeatureStore implements LDFeatureStore {
   }
 
   init(allData: LDFeatureStoreDataStorage, callback: () => void): void {
+    this.applyChanges(true, allData, undefined, callback);
+  }
+
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void {
     callback();
   }
 

packages/shared/sdk-server-edge/src/utils/mockFeatureStore.ts

@@ -6,6 +6,7 @@ const mockFeatureStore: LDFeatureStore = {
   init: jest.fn(),
   initialized: jest.fn(),
   upsert: jest.fn(),
+  applyChanges: jest.fn(),
   get: jest.fn(),
   delete: jest.fn(),
 };

packages/shared/sdk-server/__tests__/data_sources/createPayloadListenersFDv2.test.ts

@@ -88,6 +88,7 @@ describe('createPayloadListenerFDv2', () => {
     dataSourceUpdates = {
       init: jest.fn(),
       upsert: jest.fn(),
+      applyChanges: jest.fn(),
     };
     basisRecieved = jest.fn();
   });

packages/shared/sdk-server/__tests__/data_sources/createStreamListeners.test.ts

@@ -52,6 +52,7 @@ describe('createStreamListeners', () => {
     dataSourceUpdates = {
       init: jest.fn(),
       upsert: jest.fn(),
+      applyChanges: jest.fn(),
     };
     onPutCompleteHandler = jest.fn();
     onPatchCompleteHandler = jest.fn();

packages/shared/sdk-server/src/api/subsystems/LDDataSourceUpdates.ts

@@ -40,4 +40,21 @@ export interface LDDataSourceUpdates {
    *   Will be called after the upsert operation is complete.
    */
   upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void;
+
+  /**
+   * @param basis If true, completely overwrites the current contents of the data store
+   * with the provided data.  If false, upserts the items in the provided data.  Upserts
+   * are made only if provided items have newer versions than existing items.
+   * @param data An object in which each key is the "namespace" of a collection (e.g. `"features"`) and
+   * the value is an object that maps keys to entities. The actual type of this parameter is
+   * `interfaces.FullDataSet<VersionedData>`.
+   * @param selector TODO
+   * @param callback Will be called after the changes are applied.
+   */
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String,
+    callback: () => void,
+  ): void;
 }

packages/shared/sdk-server/src/api/subsystems/LDFeatureStore.ts

@@ -137,6 +137,23 @@ export interface LDFeatureStore {
    */
   upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void;
 
+  /**
+   * @param basis If true, completely overwrites the current contents of the data store
+   * with the provided data.  If false, upserts the items in the provided data.  Upserts
+   * are made only if provided items have newer versions than existing items.
+   * @param data An object in which each key is the "namespace" of a collection (e.g. `"features"`) and
+   * the value is an object that maps keys to entities. The actual type of this parameter is
+   * `interfaces.FullDataSet<VersionedData>`.
+   * @param selector TODO
+   * @param callback Will be called after the changes are applied.
+   */
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void;
+
   /**
    * Tests whether the store is initialized.
    *

packages/shared/sdk-server/src/data_sources/DataSourceUpdates.ts

@@ -67,14 +67,38 @@ export default class DataSourceUpdates implements LDDataSourceUpdates {
   ) {}
 
   init(allData: LDFeatureStoreDataStorage, callback: () => void): void {
+    this.applyChanges(true, allData, undefined, callback); // basis is true for init. selector is undefined for FDv1 init
+  }
+
+  upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void {
+    this.applyChanges(
+      false, // basis is false for upserts
+      {
+        [kind.namespace]: {
+          [data.key]: data,
+        },
+      },
+      undefined, // selector is undefined for FDv1 upsert
+      callback,
+    );
+  }
+
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void {
     const checkForChanges = this._hasEventListeners();
-    const doInit = (oldData?: LDFeatureStoreDataStorage) => {
-      this._featureStore.init(allData, () => {
+    const doApplyChanges = (oldData: LDFeatureStoreDataStorage) => {
+      this._featureStore.applyChanges(basis, data, selector, () => {
         // Defer change events so they execute after the callback.
         Promise.resolve().then(() => {
-          this._dependencyTracker.reset();
+          if (basis) {
+            this._dependencyTracker.reset();
+          }
 
-          Object.entries(allData).forEach(([namespace, items]) => {
+          Object.entries(data).forEach(([namespace, items]) => {
             Object.keys(items || {}).forEach((key) => {
               const item = items[key];
               this._dependencyTracker.updateDependenciesFrom(
@@ -87,11 +111,18 @@ export default class DataSourceUpdates implements LDDataSourceUpdates {
 
           if (checkForChanges) {
             const updatedItems = new NamespacedDataSet<boolean>();
-            Object.keys(allData).forEach((namespace) => {
-              const oldDataForKind = oldData?.[namespace] || {};
-              const newDataForKind = allData[namespace];
-              const mergedData = { ...oldDataForKind, ...newDataForKind };
-              Object.keys(mergedData).forEach((key) => {
+            Object.keys(data).forEach((namespace) => {
+              const oldDataForKind = oldData[namespace];
+              const newDataForKind = data[namespace];
+              let iterateData;
+              if (basis) {
+                // for basis, need to iterate on all keys
+                iterateData = { ...oldDataForKind, ...newDataForKind };
+              } else {
+                // for non basis, only need to iterate on keys in incoming data
+                iterateData = { ...newDataForKind };
+              }
+              Object.keys(iterateData).forEach((key) => {
                 this.addIfModified(
                   namespace,
                   key,
@@ -101,55 +132,28 @@ export default class DataSourceUpdates implements LDDataSourceUpdates {
                 );
               });
             });
+
             this.sendChangeEvents(updatedItems);
           }
         });
         callback?.();
       });
     };
 
+    let oldData = {};
     if (checkForChanges) {
+      // record old data before making changes to use for change calculations
       this._featureStore.all(VersionedDataKinds.Features, (oldFlags) => {
         this._featureStore.all(VersionedDataKinds.Segments, (oldSegments) => {
-          const oldData = {
+          oldData = {
             [VersionedDataKinds.Features.namespace]: oldFlags,
             [VersionedDataKinds.Segments.namespace]: oldSegments,
           };
-          doInit(oldData);
         });
       });
-    } else {
-      doInit();
     }
-  }
 
-  upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void {
-    const { key } = data;
-    const checkForChanges = this._hasEventListeners();
-    const doUpsert = (oldItem?: LDFeatureStoreItem | null) => {
-      this._featureStore.upsert(kind, data, () => {
-        // Defer change events so they execute after the callback.
-        Promise.resolve().then(() => {
-          this._dependencyTracker.updateDependenciesFrom(
-            kind.namespace,
-            key,
-            computeDependencies(kind.namespace, data),
-          );
-          if (checkForChanges) {
-            const updatedItems = new NamespacedDataSet<boolean>();
-            this.addIfModified(kind.namespace, key, oldItem, data, updatedItems);
-            this.sendChangeEvents(updatedItems);
-          }
-        });
-
-        callback?.();
-      });
-    };
-    if (checkForChanges) {
-      this._featureStore.get(kind, key, doUpsert);
-    } else {
-      doUpsert();
-    }
+    doApplyChanges(oldData);
   }
 
   addIfModified(

packages/shared/sdk-server/src/store/InMemoryFeatureStore.ts

@@ -12,22 +12,6 @@ export default class InMemoryFeatureStore implements LDFeatureStore {
 
   private _initCalled = false;
 
-  private _addItem(kind: DataKind, key: string, item: LDFeatureStoreItem) {
-    let items = this._allData[kind.namespace];
-    if (!items) {
-      items = {};
-      this._allData[kind.namespace] = items;
-    }
-    if (Object.hasOwnProperty.call(items, key)) {
-      const old = items[key];
-      if (!old || old.version < item.version) {
-        items[key] = item;
-      }
-    } else {
-      items[key] = item;
-    }
-  }
-
   get(kind: DataKind, key: string, callback: (res: LDFeatureStoreItem | null) => void): void {
     const items = this._allData[kind.namespace];
     if (items) {
@@ -53,19 +37,66 @@ export default class InMemoryFeatureStore implements LDFeatureStore {
   }
 
   init(allData: LDFeatureStoreDataStorage, callback: () => void): void {
-    this._initCalled = true;
-    this._allData = allData as LDFeatureStoreDataStorage;
-    callback?.();
+    this.applyChanges(true, allData, undefined, callback);
   }
 
   delete(kind: DataKind, key: string, version: number, callback: () => void): void {
-    const deletedItem = { version, deleted: true };
-    this._addItem(kind, key, deletedItem);
-    callback?.();
+    const item: LDKeyedFeatureStoreItem = { key, version, deleted: true };
+    this.applyChanges(
+      false,
+      {
+        [kind.namespace]: {
+          [key]: item,
+        },
+      },
+      undefined,
+      callback,
+    );
   }
 
   upsert(kind: DataKind, data: LDKeyedFeatureStoreItem, callback: () => void): void {
-    this._addItem(kind, data.key, data);
+    this.applyChanges(
+      false,
+      {
+        [kind.namespace]: {
+          [data.key]: data,
+        },
+      },
+      undefined,
+      callback,
+    );
+  }
+
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined, // TODO handle selector
+    callback: () => void,
+  ): void {
+    if (basis) {
+      this._initCalled = true;
+      this._allData = data;
+    } else {
+      Object.entries(data).forEach(([namespace, items]) => {
+        Object.keys(items || {}).forEach((key) => {
+          let existingItems = this._allData[namespace];
+          if (!existingItems) {
+            existingItems = {};
+            this._allData[namespace] = existingItems;
+          }
+          const item = items[key];
+          if (Object.hasOwnProperty.call(existingItems, key)) {
+            const old = existingItems[key];
+            if (!old || old.version < item.version) {
+              existingItems[key] = item;
+            }
+          } else {
+            existingItems[key] = item;
+          }
+        });
+      });
+    }
+
     callback?.();
   }
 

packages/shared/sdk-server/src/store/PersistentDataStoreWrapper.ts

@@ -252,6 +252,27 @@ export default class PersistentDataStoreWrapper implements LDFeatureStore {
     this.upsert(kind, { key, version, deleted: true }, callback);
   }
 
+  applyChanges(
+    _basis: boolean,
+    _data: LDFeatureStoreDataStorage,
+    _selector: String | undefined,
+    _callback: () => void,
+  ): void {
+    // TODO: SDK-1029 - Transactional persistent store - update this to not iterate over items and instead send data to underlying PersistentDataStore
+    // no need for queue at the moment as init and upsert handle that, but as part of SDK-1029, queue may be needed
+    if (_basis) {
+      this.init(_data, _callback);
+    } else {
+      Object.entries(_data).forEach(([namespace, items]) => {
+        Object.keys(items || {}).forEach((key) => {
+          const item = items[key];
+          this.upsert({ namespace }, { key, ...item }, () => {});
+        });
+      });
+      _callback();
+    }
+  }
+
   close(): void {
     this._itemCache?.close();
     this._allItemsCache?.close();

packages/store/node-server-sdk-dynamodb/src/DynamoDBFeatureStore.ts

@@ -51,6 +51,15 @@ export default class DynamoDBFeatureStore implements LDFeatureStore {
     this._wrapper.upsert(kind, data, callback);
   }
 
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void {
+    this._wrapper.applyChanges(basis, data, selector, callback);
+  }
+
   initialized(callback: (isInitialized: boolean) => void): void {
     this._wrapper.initialized(callback);
   }

packages/store/node-server-sdk-redis/src/RedisFeatureStore.ts

@@ -51,6 +51,15 @@ export default class RedisFeatureStore implements LDFeatureStore {
     this._wrapper.upsert(kind, data, callback);
   }
 
+  applyChanges(
+    basis: boolean,
+    data: LDFeatureStoreDataStorage,
+    selector: String | undefined,
+    callback: () => void,
+  ): void {
+    this._wrapper.applyChanges(basis, data, selector, callback);
+  }
+
   initialized(callback: (isInitialized: boolean) => void): void {
     this._wrapper.initialized(callback);
   }