From cf057df27f63db9686529e454f3abf3cc73f85e2 Mon Sep 17 00:00:00 2001 From: Martynas Kazlauskas Date: Fri, 6 Dec 2024 17:49:33 +0200 Subject: [PATCH] refactor: replace pouchdb indexeddb storage with extension storage --- .../extension-blob-collection-store.ts | 39 +++++ .../storage/extension-blob-key-value-store.ts | 61 ++++++++ .../storage/extension-document-store.ts | 74 +++++++++ .../background/storage/extension-store.ts | 20 +++ .../scripts/background/storage/migrations.ts | 109 +++++++++++++ .../src/lib/scripts/background/wallet.ts | 144 +++++++++++++++--- .../providers/ExperimentsProvider/config.ts | 7 +- .../providers/ExperimentsProvider/types.ts | 3 +- .../client/PostHogClient.ts | 3 +- 9 files changed, 434 insertions(+), 26 deletions(-) create mode 100644 apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-collection-store.ts create mode 100644 apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-key-value-store.ts create mode 100644 apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-document-store.ts create mode 100644 apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-store.ts create mode 100644 apps/browser-extension-wallet/src/lib/scripts/background/storage/migrations.ts diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-collection-store.ts b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-collection-store.ts new file mode 100644 index 000000000..cf90ec21c --- /dev/null +++ b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-collection-store.ts @@ -0,0 +1,39 @@ +/* eslint-disable @typescript-eslint/ban-types, brace-style */ +import { ExtensionDocumentStore } from './extension-document-store'; +import { storage as sdkStorage } from '@cardano-sdk/wallet'; +import { Logger } from 'ts-log'; +import { concat, defaultIfEmpty, EMPTY, filter, map, mergeMap, Observable, of } from 'rxjs'; +import { isNotNil } from '@cardano-sdk/util'; + +/** + * Stores entire collection in a single document + */ +export class ExtensionBlobCollectionStore + extends ExtensionDocumentStore + implements sdkStorage.CollectionStore +{ + /** + * @param collectionName used as extension storage key + */ + constructor(collectionName: string, logger: Logger) { + super(collectionName, logger); + } + + observeAll(): Observable { + return concat( + this.get().pipe(defaultIfEmpty([])), + this.documentChange$.pipe( + map(({ newValue }) => newValue), + filter(isNotNil) + ) + ); + } + + getAll(): Observable { + return this.get().pipe(mergeMap((items) => (items.length > 0 ? of(items) : EMPTY))); + } + + setAll(docs: T[]): Observable { + return this.set(docs); + } +} diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-key-value-store.ts b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-key-value-store.ts new file mode 100644 index 000000000..125533a26 --- /dev/null +++ b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-blob-key-value-store.ts @@ -0,0 +1,61 @@ +/* eslint-disable @typescript-eslint/ban-types, brace-style */ +import { ExtensionDocumentStore } from './extension-document-store'; +import { storage as sdkStorage } from '@cardano-sdk/wallet'; +import { Logger } from 'ts-log'; +import { EMPTY, mergeMap, Observable, of } from 'rxjs'; +import { OpaqueString } from '@cardano-sdk/util'; + +/** + * Stores entire key-value collection in a single document + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export class ExtensionBlobKeyValueStore, V extends {}> + extends ExtensionDocumentStore> + implements sdkStorage.KeyValueStore +{ + /** + * @param collectionName used as extension storage key + */ + constructor(collectionName: string, logger: Logger) { + super(collectionName, logger); + } + + getValues(keys: K[]): Observable { + return this.get().pipe( + mergeMap((collection): Observable => { + const values: V[] = []; + for (const key of keys) { + const value = collection[key]; + if (!value) { + this.logger.debug(`Key "$${key}" was not found`); + return EMPTY; + } + } + return of(values); + }) + ); + } + + setValue(key: K, value: V): Observable { + return this.get().pipe( + mergeMap((collection) => + this.set({ + ...collection, + [key]: value + }) + ) + ); + } + + setAll(docs: sdkStorage.KeyValueCollection[]): Observable { + return this.set( + docs.reduce( + (collection, { key, value }) => { + collection[key] = value; + return collection; + }, + {} as Record + ) + ); + } +} diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-document-store.ts b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-document-store.ts new file mode 100644 index 000000000..fee0dee8f --- /dev/null +++ b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-document-store.ts @@ -0,0 +1,74 @@ +/* eslint-disable @typescript-eslint/ban-types */ +import { storage as sdkStorage } from '@cardano-sdk/wallet'; +import { EMPTY, filter, from, map, mergeMap, Observable, of, share } from 'rxjs'; +import { Logger } from 'ts-log'; +import { contextLogger, fromSerializableObject, toSerializableObject } from '@cardano-sdk/util'; +import { ExtensionStore } from './extension-store'; + +export type DocumentChange = { + oldValue?: T; + newValue?: T; +}; + +const undefinedIfEmpty = (value: T | undefined): T | undefined => { + if (typeof value === 'object' && (value === null || Object.keys(value).length === 0)) return undefined; + // eslint-disable-next-line consistent-return + return value; +}; + +export class ExtensionDocumentStore extends ExtensionStore implements sdkStorage.DocumentStore { + // TODO: remove this when moving the interface to lace-platform,, it's not used + public destroyed: boolean; + + protected documentChange$: Observable>; + + // used to serialize the writes + private idle: Promise = Promise.resolve(); + + /** + * @param docId unique document id within the store, used as extension storage key + */ + constructor( + protected docId: string, + logger: Logger + ) { + super(contextLogger(logger, `ExtensionStore(${docId})`)); + this.documentChange$ = this.storageChange$.pipe( + filter(({ key }) => key === docId), + map( + ({ change }): DocumentChange => ({ + oldValue: undefinedIfEmpty(change.oldValue), + newValue: undefinedIfEmpty(change.newValue) + }) + ), + share() + ); + } + + get(): Observable { + return from(this.storage.get(this.docId)).pipe( + mergeMap((values) => { + const value = values[this.docId]; + this.logger.debug('get', value); + return value ? of(fromSerializableObject(value)) : EMPTY; + }) + ); + } + + set(doc: T): Observable { + this.logger.debug('set', doc); + return from( + (this.idle = this.idle.then(() => + this.storage.set({ + [this.docId]: toSerializableObject(doc) + }) + )) + ); + } + + destroy(): Observable { + this.destroyed = true; + // eslint-disable-next-line unicorn/no-null + return this.set(null); + } +} diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-store.ts b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-store.ts new file mode 100644 index 000000000..615e1b138 --- /dev/null +++ b/apps/browser-extension-wallet/src/lib/scripts/background/storage/extension-store.ts @@ -0,0 +1,20 @@ +/* eslint-disable @typescript-eslint/ban-types */ +import { storage as extensionStorage, Storage } from 'webextension-polyfill'; +import { fromEventPattern, mergeMap, Observable, share } from 'rxjs'; +import { Logger } from 'ts-log'; + +export abstract class ExtensionStore { + protected readonly storageChange$: Observable<{ key: string; change: Storage.StorageChange }>; + protected readonly storage: Storage.StorageArea; + + constructor(protected logger: Logger) { + this.storage = extensionStorage.local; + this.storageChange$ = fromEventPattern( + (handler) => this.storage.onChanged.addListener(handler), + (handler) => this.storage.onChanged.removeListener(handler) + ).pipe( + mergeMap((changes) => Object.entries(changes).map(([key, change]) => ({ key, change }))), + share() + ); + } +} diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/storage/migrations.ts b/apps/browser-extension-wallet/src/lib/scripts/background/storage/migrations.ts new file mode 100644 index 000000000..659cd69aa --- /dev/null +++ b/apps/browser-extension-wallet/src/lib/scripts/background/storage/migrations.ts @@ -0,0 +1,109 @@ +/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/ban-types */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types, unicorn/no-null */ +import { contextLogger } from '@cardano-sdk/util'; +import { storage } from '@cardano-sdk/wallet'; +import { defaultIfEmpty, firstValueFrom, map, Observable } from 'rxjs'; +import { Logger } from 'ts-log'; + +const migrateStore = async ( + fromStore: Store, + toStore: Store, + get: (store: Store) => Observable, + set: (store: Store, value: T) => Observable, + logger: Logger + // eslint-disable-next-line max-params +) => { + const toValue = await firstValueFrom(get(toStore)); + if (toValue) { + logger.debug('Skip store migration: already migrated'); + return false; + } + const fromValue = await firstValueFrom(get(fromStore)); + if (!fromValue) { + logger.debug('Skip store migration: no data found in source'); + return false; + } + await firstValueFrom(set(toStore, fromValue)); + logger.info('Migrated store'); + return true; +}; + +const migrateDocumentStore = async ( + fromStore: storage.DocumentStore, + toStore: storage.DocumentStore, + logger: Logger +) => + migrateStore( + fromStore, + toStore, + (store) => store.get().pipe(defaultIfEmpty(null)), + (store, value) => store.set(value), + logger + ); + +export const migrateCollectionStore = async ( + fromStore: storage.CollectionStore, + toStore: storage.CollectionStore, + logger: Logger +) => + migrateStore( + fromStore, + toStore, + (store) => + store.getAll().pipe( + map((items) => (items.length > 0 ? items : null)), + defaultIfEmpty(null) + ), + (store, items) => store.setAll(items), + logger + ); + +export const shouldAttemptWalletStoresMigration = async (stores: storage.WalletStores) => { + const tip = await firstValueFrom(stores.tip.get().pipe(defaultIfEmpty(null))); + return !tip; +}; + +export const migrateWalletStores = async ( + fromStores: storage.WalletStores, + toStores: storage.WalletStores, + baseLogger: Logger +): Promise => { + const logger = contextLogger(baseLogger, 'StorageMigration'); + if (!(await migrateDocumentStore(fromStores.tip, toStores.tip, contextLogger(logger, 'tip')))) { + logger.debug('Appears to be already migrated'); + return false; + } + await migrateDocumentStore( + fromStores.protocolParameters, + toStores.protocolParameters, + contextLogger(logger, 'protocolParameters') + ); + await migrateDocumentStore( + fromStores.genesisParameters, + toStores.genesisParameters, + contextLogger(logger, 'genesisParameters') + ); + await migrateDocumentStore(fromStores.eraSummaries, toStores.eraSummaries, contextLogger(logger, 'eraSummaries')); + await migrateDocumentStore(fromStores.assets, toStores.assets, contextLogger(logger, 'assets')); + await migrateDocumentStore(fromStores.addresses, toStores.addresses, contextLogger(logger, 'addresses')); + await migrateDocumentStore( + fromStores.inFlightTransactions, + toStores.inFlightTransactions, + contextLogger(logger, 'inFlightTransactions') + ); + await migrateDocumentStore( + fromStores.volatileTransactions, + toStores.volatileTransactions, + contextLogger(logger, 'volatileTransactions') + ); + await migrateDocumentStore(fromStores.policyIds, toStores.policyIds, contextLogger(logger, 'policyIds')); + await migrateDocumentStore( + fromStores.signedTransactions, + toStores.signedTransactions, + contextLogger(logger, 'signedTransactions') + ); + + await migrateCollectionStore(fromStores.transactions, toStores.transactions, contextLogger(logger, 'transactions')); + await migrateCollectionStore(fromStores.utxo, toStores.utxo, contextLogger(logger, 'utxo')); + return true; +}; diff --git a/apps/browser-extension-wallet/src/lib/scripts/background/wallet.ts b/apps/browser-extension-wallet/src/lib/scripts/background/wallet.ts index 600a6318d..e50aca9f5 100644 --- a/apps/browser-extension-wallet/src/lib/scripts/background/wallet.ts +++ b/apps/browser-extension-wallet/src/lib/scripts/background/wallet.ts @@ -1,5 +1,6 @@ +/* eslint-disable unicorn/no-null */ import { runtime, storage as webStorage } from 'webextension-polyfill'; -import { of, combineLatest, map, EMPTY, BehaviorSubject } from 'rxjs'; +import { of, combineLatest, map, EMPTY, BehaviorSubject, Observable, from, firstValueFrom, defaultIfEmpty } from 'rxjs'; import { getProviders } from './config'; import { DEFAULT_LOOK_AHEAD_SEARCH, @@ -37,27 +38,18 @@ import { logger } from '@lace/common'; import { getBackgroundStorage } from '@lib/scripts/background/storage'; import { ExperimentName } from '@providers/ExperimentsProvider/types'; import { requestMessage$ } from './services/utilityServices'; -import { MessageTypes } from '../types'; +import { BackgroundStorage, MessageTypes } from '../types'; +import { ExtensionDocumentStore } from './storage/extension-document-store'; +import { ExtensionBlobKeyValueStore } from './storage/extension-blob-key-value-store'; +import { ExtensionBlobCollectionStore } from './storage/extension-blob-collection-store'; +import { migrateCollectionStore, migrateWalletStores, shouldAttemptWalletStoresMigration } from './storage/migrations'; if (typeof window !== 'undefined') { throw new TypeError('This module should only be imported in service worker'); } -// It is important that this file is not exported from index, -// because creating wallet repository with store creates an actual pouchdb database -// which results in some trash files when running the tests (leveldb directory) -export const walletRepository = new WalletRepository({ - logger, - store$: of( - new Wallet.storage.PouchDbCollectionStore>( - { dbName: 'walletRepository', computeDocId: (wallet) => wallet.walletId }, - logger - ) - ) -}); - -const chainIdToChainName = (chainId: Cardano.ChainId): Wallet.ChainName => { - switch (chainId.networkMagic) { +const networkMagicToChainName = (networkMagic: Cardano.NetworkMagic): Wallet.ChainName => { + switch (networkMagic) { case Wallet.Cardano.ChainIds.Mainnet.networkMagic: return 'Mainnet'; case Wallet.Cardano.ChainIds.Preprod.networkMagic: @@ -67,16 +59,62 @@ const chainIdToChainName = (chainId: Cardano.ChainId): Wallet.ChainName => { case Wallet.Cardano.ChainIds.Sanchonet.networkMagic: return 'Sanchonet'; default: - throw new Error(`Unknown network magic: ${chainId.networkMagic}`); + throw new Error(`Unknown network magic: ${networkMagic}`); } }; +type FeatureFlags = BackgroundStorage['featureFlags'][0]; +const isExperimentEnabled = (featureFlags: FeatureFlags, experimentName: ExperimentName) => + // TODO: delete hardcoded 'true' before merging + experimentName === ExperimentName.EXTENSION_STORAGE ? true : !!(featureFlags?.[experimentName] ?? false); + +const getFeatureFlags = async (networkMagic: Cardano.NetworkMagic) => { + const chainName = networkMagicToChainName(networkMagic); + const backgroundStorage = await getBackgroundStorage(); + const magic = getMagicForChain(chainName); + return backgroundStorage?.featureFlags?.[magic]; +}; + +const createPouchdbWalletRepositoryStore = () => + new Wallet.storage.PouchDbCollectionStore>( + { dbName: 'walletRepository', computeDocId: (wallet) => wallet.walletId }, + logger + ); + +type StoredWallet = AnyWallet; +const createWalletRepositoryStore = (): Observable> => + from( + (async () => { + // wallet repository is always using feature flag of 'mainnet', because it has to be the same for all networks + const featureFlags = await getFeatureFlags(Wallet.Cardano.ChainIds.Mainnet.networkMagic); + if (isExperimentEnabled(featureFlags, ExperimentName.EXTENSION_STORAGE)) { + const extensionStore = new ExtensionBlobCollectionStore('walletRepository', logger); + const wallets = await firstValueFrom(extensionStore.getAll().pipe(defaultIfEmpty(null))); + if (!wallets) { + const pouchdbStore = createPouchdbWalletRepositoryStore(); + await migrateCollectionStore(pouchdbStore, extensionStore, logger); + } + return extensionStore; + } + + return createPouchdbWalletRepositoryStore(); + })() + ); + +// It is important that this file is not exported from index, +// because creating wallet repository with store creates an actual pouchdb database +// which results in some trash files when running the tests (leveldb directory) +export const walletRepository = new WalletRepository({ + logger, + store$: createWalletRepositoryStore() +}); + // eslint-disable-next-line unicorn/no-null const currentWalletProviders$ = new BehaviorSubject(null); const walletFactory: WalletFactory = { create: async ({ chainId, accountIndex }, wallet, { stores, witnesser }) => { - const chainName: Wallet.ChainName = chainIdToChainName(chainId); + const chainName: Wallet.ChainName = networkMagicToChainName(chainId.networkMagic); const providers = await getProviders(chainName); // Caches current wallet providers. @@ -134,9 +172,8 @@ const walletFactory: WalletFactory name.replace(/[^\da-z]/gi, ''); -const storesFactory: StoresFactory = { +const pouchdbStoresFactory: StoresFactory = { create: async ({ name }) => { const baseDbName = getBaseDbName(name); const docsDbName = `${baseDbName}Docs`; @@ -226,6 +263,67 @@ const storesFactory: StoresFactory = { } }; +export const extensionStorageStoresFactory: StoresFactory = { + create: async ({ name }) => ({ + addresses: new ExtensionDocumentStore(`${name}_addresses`, logger), + assets: new ExtensionDocumentStore(`${name}_assets`, logger), + destroy() { + if (!this.destroyed) { + // since the database of document stores is shared, destroying any document store destroys all of them + this.destroyed = true; + logger.warn('Destroying wallet stores...'); + const destroyDocumentsDb = this.tip.destroy(); + return combineLatest([ + destroyDocumentsDb, + this.transactions.destroy(), + this.utxo.destroy(), + this.unspendableUtxo.destroy(), + this.rewardsHistory.destroy(), + this.stakePools.destroy(), + this.rewardsBalances.destroy() + ]).pipe(map(() => void 0)); + } + return EMPTY; + }, + destroyed: false, + eraSummaries: new ExtensionDocumentStore(`${name}_eraSummaries`, logger), + genesisParameters: new ExtensionDocumentStore(`${name}_genesisParameters`, logger), + inFlightTransactions: new ExtensionDocumentStore(`${name}_transactionsInFlight`, logger), + policyIds: new ExtensionDocumentStore(`${name}_handlePolicyIds`, logger), + protocolParameters: new ExtensionDocumentStore(`${name}_protocolParameters`, logger), + rewardsBalances: new ExtensionBlobKeyValueStore(`${name}_rewardsBalances`, logger), + rewardsHistory: new ExtensionBlobKeyValueStore(`${name}_rewardsHistory`, logger), + stakePools: new ExtensionBlobKeyValueStore(`${name}_stakePools`, logger), + signedTransactions: new ExtensionDocumentStore(`${name}_signedTransactions`, logger), + tip: new ExtensionDocumentStore(`${name}_tip`, logger), + transactions: new ExtensionBlobCollectionStore(`${name}_transactions`, logger), + unspendableUtxo: new ExtensionBlobCollectionStore(`${name}_unspendableUtxo`, logger), + utxo: new ExtensionBlobCollectionStore(`${name}_utxo`, logger), + volatileTransactions: new ExtensionDocumentStore(`${name}_volatileTransactions`, logger) + }) +}; + +// Used for migrations to get feature flags, which are enabled per chainId +// This is coupled with WalletManager implementation (getWalletStoreId function) +const getNetworkMagic = (storeName: string) => Number.parseInt(storeName.split('-')[1]) as Cardano.NetworkMagic; + +const storesFactory: StoresFactory = { + async create(props) { + const featureFlags = await getFeatureFlags(getNetworkMagic(props.name)); + if (isExperimentEnabled(featureFlags, ExperimentName.EXTENSION_STORAGE)) { + const extensionStores = await extensionStorageStoresFactory.create(props); + if (await shouldAttemptWalletStoresMigration(extensionStores)) { + const pouchdbStores = await pouchdbStoresFactory.create(props); + if (await migrateWalletStores(pouchdbStores, extensionStores, logger)) { + // TODO: consider destroying pouchdb stores on successful migration + } + } + return extensionStores; + } + return pouchdbStoresFactory.create(props); + } +}; + const signingCoordinatorApi = consumeSigningCoordinatorApi({ logger, runtime }); export const walletManager = new WalletManager( { name: process.env.WALLET_NAME }, diff --git a/apps/browser-extension-wallet/src/providers/ExperimentsProvider/config.ts b/apps/browser-extension-wallet/src/providers/ExperimentsProvider/config.ts index 7f0c95f33..194d03942 100644 --- a/apps/browser-extension-wallet/src/providers/ExperimentsProvider/config.ts +++ b/apps/browser-extension-wallet/src/providers/ExperimentsProvider/config.ts @@ -6,7 +6,8 @@ export const getDefaultFeatureFlags = (): FallbackConfiguration => ({ [ExperimentName.USE_SWITCH_TO_NAMI_MODE]: false, [ExperimentName.SHARED_WALLETS]: false, [ExperimentName.WEBSOCKET_API]: false, - [ExperimentName.BLOCKFROST_ASSET_PROVIDER]: false + [ExperimentName.BLOCKFROST_ASSET_PROVIDER]: false, + [ExperimentName.EXTENSION_STORAGE]: false }); export const experiments: ExperimentsConfig = { @@ -33,5 +34,9 @@ export const experiments: ExperimentsConfig = { [ExperimentName.BLOCKFROST_ASSET_PROVIDER]: { value: false, default: false + }, + [ExperimentName.EXTENSION_STORAGE]: { + value: false, + default: false } }; diff --git a/apps/browser-extension-wallet/src/providers/ExperimentsProvider/types.ts b/apps/browser-extension-wallet/src/providers/ExperimentsProvider/types.ts index 37d088b09..60c625700 100644 --- a/apps/browser-extension-wallet/src/providers/ExperimentsProvider/types.ts +++ b/apps/browser-extension-wallet/src/providers/ExperimentsProvider/types.ts @@ -11,7 +11,8 @@ export enum ExperimentName { USE_SWITCH_TO_NAMI_MODE = 'use-switch-to-nami-mode', SHARED_WALLETS = 'shared-wallets', WEBSOCKET_API = 'websocket-api', - BLOCKFROST_ASSET_PROVIDER = 'blockfrost-asset-provider' + BLOCKFROST_ASSET_PROVIDER = 'blockfrost-asset-provider', + EXTENSION_STORAGE = 'extension-storage' } interface FeatureFlag { diff --git a/apps/browser-extension-wallet/src/providers/PostHogClientProvider/client/PostHogClient.ts b/apps/browser-extension-wallet/src/providers/PostHogClientProvider/client/PostHogClient.ts index f9dbd62a3..d9863d10f 100644 --- a/apps/browser-extension-wallet/src/providers/PostHogClientProvider/client/PostHogClient.ts +++ b/apps/browser-extension-wallet/src/providers/PostHogClientProvider/client/PostHogClient.ts @@ -31,7 +31,8 @@ type FeatureFlag = | 'shared-wallets' | 'use-switch-to-nami-mode' | 'websocket-api' - | ExperimentName.BLOCKFROST_ASSET_PROVIDER; + | ExperimentName.BLOCKFROST_ASSET_PROVIDER + | ExperimentName.EXTENSION_STORAGE; type FeatureFlags = { [key in FeatureFlag]: boolean;