diff --git a/jestSetup.js b/jestSetup.js index 2288af999..0a5ef85fb 100644 --- a/jestSetup.js +++ b/jestSetup.js @@ -1,7 +1,7 @@ jest.mock('./lib/storage'); -jest.mock('./lib/storage/NativeStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/WebStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/providers/IDBKeyVal', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/index.native', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/index', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/providers/IDBKeyValProvider', () => require('./lib/storage/__mocks__')); jest.mock('react-native-device-info', () => ({getFreeDiskStorage: () => {}})); jest.mock('react-native-quick-sqlite', () => ({ diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 3d3992532..507de756d 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -38,6 +38,16 @@ function init({ shouldSyncMultipleInstances = Boolean(global.localStorage), debugSetState = false, }: InitOptions): void { + Storage.init(); + + if (shouldSyncMultipleInstances) { + Storage.keepInstancesSync?.((key, value) => { + const prevValue = cache.getValue(key, false); + cache.set(key, value); + OnyxUtils.keyChanged(key, value, prevValue); + }); + } + if (debugSetState) { PerformanceUtils.setShouldDebugSetState(true); } @@ -50,14 +60,6 @@ function init({ // Initialize all of our keys with data provided then give green light to any pending connections Promise.all([OnyxUtils.addAllSafeEvictionKeysToRecentlyAccessedList(), OnyxUtils.initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); - - if (shouldSyncMultipleInstances) { - Storage.keepInstancesSync?.((key, value) => { - const prevValue = cache.getValue(key, false); - cache.set(key, value); - OnyxUtils.keyChanged(key, value, prevValue); - }); - } } /** diff --git a/lib/storage/InstanceSync/index.ts b/lib/storage/InstanceSync/index.ts new file mode 100644 index 000000000..543cc0409 --- /dev/null +++ b/lib/storage/InstanceSync/index.ts @@ -0,0 +1,17 @@ +import NOOP from 'lodash/noop'; + +/** + * This is used to keep multiple browser tabs in sync, therefore only needed on web + * On native platforms, we omit this syncing logic by setting this to mock implementation. + */ +const InstanceSync = { + shouldBeUsed: false, + init: NOOP, + setItem: NOOP, + removeItem: NOOP, + removeItems: NOOP, + mergeItem: NOOP, + clear: void>(callback: T) => Promise.resolve(callback()), +}; + +export default InstanceSync; diff --git a/lib/storage/InstanceSync/index.web.ts b/lib/storage/InstanceSync/index.web.ts new file mode 100644 index 000000000..6be21b2e7 --- /dev/null +++ b/lib/storage/InstanceSync/index.web.ts @@ -0,0 +1,72 @@ +/** + * The InstancesSync object provides data-changed events like the ones that exist + * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when + * data changes and then stay up-to-date with everything happening in Onyx. + */ +import type {OnyxKey} from '../../types'; +import NoopProvider from '../providers/NoopProvider'; +import type {KeyList, OnStorageKeyChanged} from '../providers/types'; +import type StorageProvider from '../providers/types'; + +const SYNC_ONYX = 'SYNC_ONYX'; + +/** + * Raise an event through `localStorage` to let other tabs know a value changed + * @param {String} onyxKey + */ +function raiseStorageSyncEvent(onyxKey: OnyxKey) { + global.localStorage.setItem(SYNC_ONYX, onyxKey); + global.localStorage.removeItem(SYNC_ONYX); +} + +function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { + onyxKeys.forEach((onyxKey) => { + raiseStorageSyncEvent(onyxKey); + }); +} + +let storage = NoopProvider; + +const InstanceSync = { + shouldBeUsed: true, + /** + * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync + */ + init: (onStorageKeyChanged: OnStorageKeyChanged, store: StorageProvider) => { + storage = store; + + // This listener will only be triggered by events coming from other tabs + global.addEventListener('storage', (event) => { + // Ignore events that don't originate from the SYNC_ONYX logic + if (event.key !== SYNC_ONYX || !event.newValue) { + return; + } + + const onyxKey = event.newValue; + + storage.getItem(onyxKey).then((value) => onStorageKeyChanged(onyxKey, value)); + }); + }, + setItem: raiseStorageSyncEvent, + removeItem: raiseStorageSyncEvent, + removeItems: raiseStorageSyncManyKeysEvent, + mergeItem: raiseStorageSyncEvent, + clear: (clearImplementation: () => void) => { + let allKeys: KeyList; + + // The keys must be retrieved before storage is cleared or else the list of keys would be empty + return storage + .getAllKeys() + .then((keys: KeyList) => { + allKeys = keys; + }) + .then(() => clearImplementation()) + .then(() => { + // Now that storage is cleared, the storage sync event can happen which is a more atomic action + // for other browser tabs + raiseStorageSyncManyKeysEvent(allKeys); + }); + }, +}; + +export default InstanceSync; diff --git a/lib/storage/NativeStorage.ts b/lib/storage/NativeStorage.ts deleted file mode 100644 index 1473613fa..000000000 --- a/lib/storage/NativeStorage.ts +++ /dev/null @@ -1,3 +0,0 @@ -import SQLiteStorage from './providers/SQLiteStorage'; - -export default SQLiteStorage; diff --git a/lib/storage/WebStorage.ts b/lib/storage/WebStorage.ts deleted file mode 100644 index 6621a084b..000000000 --- a/lib/storage/WebStorage.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * This file is here to wrap IDBKeyVal with a layer that provides data-changed events like the ones that exist - * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when - * data changes and then stay up-to-date with everything happening in Onyx. - */ -import type {OnyxKey} from '../types'; -import Storage from './providers/IDBKeyVal'; -import type {KeyList} from './providers/types'; -import type StorageProvider from './providers/types'; - -const SYNC_ONYX = 'SYNC_ONYX'; - -/** - * Raise an event thorough `localStorage` to let other tabs know a value changed - */ -function raiseStorageSyncEvent(onyxKey: OnyxKey) { - global.localStorage.setItem(SYNC_ONYX, onyxKey); - global.localStorage.removeItem(SYNC_ONYX); -} - -function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { - onyxKeys.forEach((onyxKey) => { - raiseStorageSyncEvent(onyxKey); - }); -} - -const webStorage: StorageProvider = { - ...Storage, - /** - * @param onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync - */ - keepInstancesSync(onStorageKeyChanged) { - // Override set, remove and clear to raise storage events that we intercept in other tabs - this.setItem = (key, value) => Storage.setItem(key, value).then(() => raiseStorageSyncEvent(key)); - - this.removeItem = (key) => Storage.removeItem(key).then(() => raiseStorageSyncEvent(key)); - - this.removeItems = (keys) => Storage.removeItems(keys).then(() => raiseStorageSyncManyKeysEvent(keys)); - - this.mergeItem = (key, batchedChanges, modifiedData) => Storage.mergeItem(key, batchedChanges, modifiedData).then(() => raiseStorageSyncEvent(key)); - - // If we just call Storage.clear other tabs will have no idea which keys were available previously - // so that they can call keysChanged for them. That's why we iterate over every key and raise a storage sync - // event for each one - this.clear = () => { - let allKeys: KeyList; - - // The keys must be retrieved before storage is cleared or else the list of keys would be empty - return Storage.getAllKeys() - .then((keys) => { - allKeys = keys; - }) - .then(() => Storage.clear()) - .then(() => { - // Now that storage is cleared, the storage sync event can happen which is a more atomic action - // for other browser tabs - allKeys.forEach(raiseStorageSyncEvent); - }); - }; - - // This listener will only be triggered by events coming from other tabs - global.addEventListener('storage', (event) => { - // Ignore events that don't originate from the SYNC_ONYX logic - if (event.key !== SYNC_ONYX || !event.newValue) { - return; - } - - const onyxKey = event.newValue; - Storage.getItem(onyxKey).then((value) => onStorageKeyChanged(onyxKey, value)); - }); - }, -}; - -export default webStorage; diff --git a/lib/storage/__mocks__/index.ts b/lib/storage/__mocks__/index.ts index a88e6f953..3a15fecf1 100644 --- a/lib/storage/__mocks__/index.ts +++ b/lib/storage/__mocks__/index.ts @@ -1,89 +1,26 @@ -import type {OnyxKey, OnyxValue} from '../../types'; -import utils from '../../utils'; -import type {KeyValuePairList} from '../providers/types'; -import type StorageProvider from '../providers/types'; +import MemoryOnlyProvider, {mockStore, mockSet, setMockStore} from '../providers/MemoryOnlyProvider'; -let storageMapInternal: Record> = {}; +const init = jest.fn(MemoryOnlyProvider.init); -const set = jest.fn((key, value) => { - storageMapInternal[key] = value; - return Promise.resolve(value); -}); +init(); -const idbKeyvalMock: StorageProvider = { - setItem(key, value) { - return set(key, value); - }, - multiSet(pairs) { - const setPromises = pairs.map(([key, value]) => this.setItem(key, value)); - return new Promise((resolve) => { - Promise.all(setPromises).then(() => resolve(storageMapInternal)); - }); - }, - getItem(key: TKey) { - return Promise.resolve(storageMapInternal[key] as OnyxValue); - }, - multiGet(keys) { - const getPromises = keys.map( - (key) => - new Promise((resolve) => { - this.getItem(key).then((value) => resolve([key, value])); - }), - ); - return Promise.all(getPromises) as Promise; - }, - multiMerge(pairs) { - pairs.forEach(([key, value]) => { - const existingValue = storageMapInternal[key]; - const newValue = utils.fastMerge(existingValue as Record, value as Record); - - set(key, newValue); - }); - - return Promise.resolve(storageMapInternal); - }, - mergeItem(key, _changes, modifiedData) { - return this.setItem(key, modifiedData); - }, - removeItem(key) { - delete storageMapInternal[key]; - return Promise.resolve(); - }, - removeItems(keys) { - keys.forEach((key) => { - delete storageMapInternal[key]; - }); - return Promise.resolve(); - }, - clear() { - storageMapInternal = {}; - return Promise.resolve(); - }, - getAllKeys() { - return Promise.resolve(Object.keys(storageMapInternal)); - }, - getDatabaseSize() { - return Promise.resolve({bytesRemaining: 0, bytesUsed: 99999}); - }, -}; - -const idbKeyvalMockSpy = { - idbKeyvalSet: set, - setItem: jest.fn(idbKeyvalMock.setItem), - getItem: jest.fn(idbKeyvalMock.getItem), - removeItem: jest.fn(idbKeyvalMock.removeItem), - removeItems: jest.fn(idbKeyvalMock.removeItems), - clear: jest.fn(idbKeyvalMock.clear), - getAllKeys: jest.fn(idbKeyvalMock.getAllKeys), - multiGet: jest.fn(idbKeyvalMock.multiGet), - multiSet: jest.fn(idbKeyvalMock.multiSet), - multiMerge: jest.fn(idbKeyvalMock.multiMerge), - mergeItem: jest.fn(idbKeyvalMock.mergeItem), - getStorageMap: jest.fn(() => storageMapInternal), - setInitialMockData: jest.fn((data) => { - storageMapInternal = data; - }), - getDatabaseSize: jest.fn(idbKeyvalMock.getDatabaseSize), +const StorageMock = { + init, + getItem: jest.fn(MemoryOnlyProvider.getItem), + multiGet: jest.fn(MemoryOnlyProvider.multiGet), + setItem: jest.fn(MemoryOnlyProvider.setItem), + multiSet: jest.fn(MemoryOnlyProvider.multiSet), + mergeItem: jest.fn(MemoryOnlyProvider.mergeItem), + multiMerge: jest.fn(MemoryOnlyProvider.multiMerge), + removeItem: jest.fn(MemoryOnlyProvider.removeItem), + removeItems: jest.fn(MemoryOnlyProvider.removeItems), + clear: jest.fn(MemoryOnlyProvider.clear), + getAllKeys: jest.fn(MemoryOnlyProvider.getAllKeys), + getDatabaseSize: jest.fn(MemoryOnlyProvider.getDatabaseSize), + keepInstancesSync: jest.fn(), + mockSet, + getMockStore: jest.fn(() => mockStore), + setMockStore: jest.fn((data) => setMockStore(data)), }; -export default idbKeyvalMockSpy; +export default StorageMock; diff --git a/lib/storage/index.native.ts b/lib/storage/index.native.ts deleted file mode 100644 index 51b21ca5a..000000000 --- a/lib/storage/index.native.ts +++ /dev/null @@ -1,3 +0,0 @@ -import NativeStorage from './NativeStorage'; - -export default NativeStorage; diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 4ee520d20..5871e7fb0 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -1,3 +1,187 @@ -import WebStorage from './WebStorage'; +import * as Logger from '../Logger'; -export default WebStorage; +import PlatformStorage from './platforms'; +import InstanceSync from './InstanceSync'; +import MemoryOnlyProvider from './providers/MemoryOnlyProvider'; +import type StorageProvider from './providers/types'; + +let provider = PlatformStorage; +let shouldKeepInstancesSync = false; +let finishInitalization: (value?: unknown) => void; +const initPromise = new Promise((resolve) => { + finishInitalization = resolve; +}); + +type Storage = { + getStorageProvider: () => StorageProvider; +} & Omit; + +/** + * Degrade performance by removing the storage provider and only using cache + */ +function degradePerformance(error: Error) { + Logger.logAlert(`Error while using ${provider.name}. Falling back to only using cache and dropping storage.`); + console.error(error); + provider = MemoryOnlyProvider; +} + +/** + * Runs a piece of code and degrades performance if certain errors are thrown + */ +function tryOrDegradePerformance(fn: () => Promise | T, waitForInitialization = true): Promise { + return new Promise((resolve, reject) => { + const promise = waitForInitialization ? initPromise : Promise.resolve(); + + promise.then(() => { + try { + resolve(fn()); + } catch (error) { + // Test for known critical errors that the storage provider throws, e.g. when storage is full + if (error instanceof Error) { + // IndexedDB error when storage is full (https://github.com/Expensify/App/issues/29403) + if (error.message.includes('Internal error opening backing store for indexedDB.open')) { + degradePerformance(error); + } + + // catch the error if DB connection can not be established/DB can not be created + if (error.message.includes('IDBKeyVal store could not be created')) { + degradePerformance(error); + } + } + + reject(error); + } + }); + }); +} + +const Storage: Storage = { + /** + * Returns the storage provider currently in use + */ + getStorageProvider() { + return provider; + }, + + /** + * Initializes all providers in the list of storage providers + * and enables fallback providers if necessary + */ + init() { + tryOrDegradePerformance(provider.init, false).finally(() => { + finishInitalization(); + }); + }, + + /** + * Get the value of a given key or return `null` if it's not available + */ + getItem: (key) => tryOrDegradePerformance(() => provider.getItem(key)), + + /** + * Get multiple key-value pairs for the give array of keys in a batch + */ + multiGet: (keys) => tryOrDegradePerformance(() => provider.multiGet(keys)), + + /** + * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string + */ + setItem: (key, value) => + tryOrDegradePerformance(() => { + const promise = provider.setItem(key, value); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.setItem(key)); + } + + return promise; + }), + + /** + * Stores multiple key-value pairs in a batch + */ + multiSet: (pairs) => tryOrDegradePerformance(() => provider.multiSet(pairs)), + + /** + * Merging an existing value with a new one + */ + mergeItem: (key, changes, modifiedData) => + tryOrDegradePerformance(() => { + const promise = provider.mergeItem(key, changes, modifiedData); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.mergeItem(key)); + } + + return promise; + }), + + /** + * Multiple merging of existing and new values in a batch + * This function also removes all nested null values from an object. + */ + multiMerge: (pairs) => tryOrDegradePerformance(() => provider.multiMerge(pairs)), + + /** + * Removes given key and its value + */ + removeItem: (key) => + tryOrDegradePerformance(() => { + const promise = provider.removeItem(key); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItem(key)); + } + + return promise; + }), + + /** + * Remove given keys and their values + */ + removeItems: (keys) => + tryOrDegradePerformance(() => { + const promise = provider.removeItems(keys); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItems(keys)); + } + + return promise; + }), + + /** + * Clears everything + */ + clear: () => + tryOrDegradePerformance(() => { + if (shouldKeepInstancesSync) { + return InstanceSync.clear(() => provider.clear()); + } + + return provider.clear(); + }), + + /** + * Returns all available keys + */ + getAllKeys: () => tryOrDegradePerformance(() => provider.getAllKeys()), + + /** + * Gets the total bytes of the store + */ + getDatabaseSize: () => tryOrDegradePerformance(() => provider.getDatabaseSize()), + + /** + * @param onStorageKeyChanged - Storage synchronization mechanism keeping all opened tabs in sync (web only) + */ + keepInstancesSync(onStorageKeyChanged) { + // If InstanceSync shouldn't be used, it means we're on a native platform and we don't need to keep instances in sync + if (!InstanceSync.shouldBeUsed) return; + + shouldKeepInstancesSync = true; + InstanceSync.init(onStorageKeyChanged, this); + }, +}; + +export default Storage; diff --git a/lib/storage/platforms/index.native.ts b/lib/storage/platforms/index.native.ts new file mode 100644 index 000000000..95822c4a5 --- /dev/null +++ b/lib/storage/platforms/index.native.ts @@ -0,0 +1,3 @@ +import NativeStorage from '../providers/SQLiteProvider'; + +export default NativeStorage; diff --git a/lib/storage/platforms/index.ts b/lib/storage/platforms/index.ts new file mode 100644 index 000000000..0b95dc97d --- /dev/null +++ b/lib/storage/platforms/index.ts @@ -0,0 +1,3 @@ +import WebStorage from '../providers/IDBKeyValProvider'; + +export default WebStorage; diff --git a/lib/storage/providers/IDBKeyVal.ts b/lib/storage/providers/IDBKeyValProvider.ts similarity index 70% rename from lib/storage/providers/IDBKeyVal.ts rename to lib/storage/providers/IDBKeyValProvider.ts index 6312d0e3e..3a385d228 100644 --- a/lib/storage/providers/IDBKeyVal.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -6,19 +6,27 @@ import type {OnyxKey, OnyxValue} from '../../types'; // We don't want to initialize the store while the JS bundle loads as idb-keyval will try to use global.indexedDB // which might not be available in certain environments that load the bundle (e.g. electron main process). -let customStoreInstance: UseStore; -function getCustomStore(): UseStore { - if (!customStoreInstance) { - customStoreInstance = createStore('OnyxDB', 'keyvaluepairs'); - } - return customStoreInstance; -} +let idbKeyValStore: UseStore; const provider: StorageProvider = { - setItem: (key, value) => set(key, value, getCustomStore()), - multiGet: (keysParam) => getMany(keysParam, getCustomStore()).then((values) => values.map((value, index) => [keysParam[index], value])), + /** + * The name of the provider that can be printed to the logs + */ + name: 'IDBKeyValProvider', + /** + * Initializes the storage provider + */ + init() { + const newIdbKeyValStore = createStore('OnyxDB', 'keyvaluepairs'); + + if (newIdbKeyValStore == null) throw Error('IDBKeyVal store could not be created'); + + idbKeyValStore = newIdbKeyValStore; + }, + setItem: (key, value) => set(key, value, idbKeyValStore), + multiGet: (keysParam) => getMany(keysParam, idbKeyValStore).then((values) => values.map((value, index) => [keysParam[index], value])), multiMerge: (pairs) => - getCustomStore()('readwrite', (store) => { + idbKeyValStore('readwrite', (store) => { // Note: we are using the manual store transaction here, to fit the read and update // of the items in one transaction to achieve best performance. const getValues = Promise.all(pairs.map(([key]) => promisifyRequest>(store.get(key)))); @@ -36,15 +44,15 @@ const provider: StorageProvider = { // Since Onyx also merged the existing value with the changes, we can just set the value directly return provider.setItem(key, modifiedData); }, - multiSet: (pairs) => setMany(pairs, getCustomStore()), - clear: () => clear(getCustomStore()), - getAllKeys: () => keys(getCustomStore()), + multiSet: (pairs) => setMany(pairs, idbKeyValStore), + clear: () => clear(idbKeyValStore), + getAllKeys: () => keys(idbKeyValStore), getItem: (key) => - get(key, getCustomStore()) + get(key, idbKeyValStore) // idb-keyval returns undefined for missing items, but this needs to return null so that idb-keyval does the same thing as SQLiteStorage. .then((val) => (val === undefined ? null : val)), - removeItem: (key) => del(key, getCustomStore()), - removeItems: (keysParam) => delMany(keysParam, getCustomStore()), + removeItem: (key) => del(key, idbKeyValStore), + removeItems: (keysParam) => delMany(keysParam, idbKeyValStore), getDatabaseSize() { if (!window.navigator || !window.navigator.storage) { throw new Error('StorageManager browser API unavailable'); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts new file mode 100644 index 000000000..74d9e3a84 --- /dev/null +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -0,0 +1,148 @@ +import _ from 'underscore'; +import utils from '../../utils'; +import type StorageProvider from './types'; +import type {KeyValuePair} from './types'; +import type {OnyxKey, OnyxValue} from '../../types'; + +type Store = Record>; + +// eslint-disable-next-line import/no-mutable-exports +let store: Store = {}; + +const setInternal = (key: OnyxKey, value: OnyxValue) => { + store[key] = value; + return Promise.resolve(value); +}; + +const isJestRunning = typeof jest !== 'undefined'; +const set = isJestRunning ? jest.fn(setInternal) : setInternal; + +const provider: StorageProvider = { + /** + * The name of the provider that can be printed to the logs + */ + name: 'MemoryOnlyProvider', + + /** + * Initializes the storage provider + */ + init() { + // do nothing + }, + + /** + * Get the value of a given key or return `null` if it's not available in memory + */ + getItem(key) { + const value = store[key] as OnyxValue; + + return Promise.resolve(value === undefined ? null : value); + }, + + /** + * Get multiple key-value pairs for the give array of keys in a batch. + */ + multiGet(keys) { + const getPromises = _.map( + keys, + (key) => + new Promise((resolve) => { + this.getItem(key).then((value) => resolve([key, value])); + }), + ) as Array>; + return Promise.all(getPromises); + }, + + /** + * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string + */ + setItem(key, value) { + set(key, value); + + return Promise.resolve(); + }, + + /** + * Stores multiple key-value pairs in a batch + */ + multiSet(pairs) { + const setPromises = _.map(pairs, ([key, value]) => this.setItem(key, value)); + + return new Promise((resolve) => { + Promise.all(setPromises).then(() => { + resolve(undefined); + }); + }); + }, + + /** + * Merging an existing value with a new one + */ + mergeItem(key, _changes, modifiedData) { + // Since Onyx already merged the existing value with the changes, we can just set the value directly + return this.setItem(key, modifiedData); + }, + + /** + * Multiple merging of existing and new values in a batch + * This function also removes all nested null values from an object. + */ + multiMerge(pairs) { + _.forEach(pairs, ([key, value]) => { + const existingValue = store[key] as Record; + const newValue = utils.fastMerge(existingValue, value as Record) as OnyxValue; + + set(key, newValue); + }); + + return Promise.resolve([]); + }, + + /** + * Remove given key and it's value from memory + */ + removeItem(key) { + delete store[key]; + return Promise.resolve(); + }, + + /** + * Remove given keys and their values from memory + */ + removeItems(keys) { + _.each(keys, (key) => { + delete store[key]; + }); + return Promise.resolve(); + }, + + /** + * Clear everything from memory + */ + clear() { + store = {}; + return Promise.resolve(); + }, + + /** + * Returns all keys available in memory + */ + getAllKeys() { + return Promise.resolve(_.keys(store)); + }, + + /** + * Gets the total bytes of the store. + * `bytesRemaining` will always be `Number.POSITIVE_INFINITY` since we don't have a hard limit on memory. + */ + getDatabaseSize() { + return Promise.resolve({bytesRemaining: Number.POSITIVE_INFINITY, bytesUsed: 0}); + }, +}; + +const setMockStore = (data: Store) => { + store = data; +}; + +export default provider; +export {store as mockStore, set as mockSet, setMockStore}; diff --git a/lib/storage/providers/NoopProvider.ts b/lib/storage/providers/NoopProvider.ts new file mode 100644 index 000000000..07af0a496 --- /dev/null +++ b/lib/storage/providers/NoopProvider.ts @@ -0,0 +1,98 @@ +import type StorageProvider from './types'; + +const provider: StorageProvider = { + /** + * The name of the provider that can be printed to the logs + */ + name: 'NoopProvider', + + /** + * Initializes the storage provider + */ + init() { + // do nothing + }, + + /** + * Get the value of a given key or return `null` if it's not available in memory + * @param {String} key + * @return {Promise<*>} + */ + getItem() { + return Promise.resolve(null); + }, + + /** + * Get multiple key-value pairs for the give array of keys in a batch. + */ + multiGet() { + return Promise.resolve([]); + }, + + /** + * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string + */ + setItem() { + return Promise.resolve(); + }, + + /** + * Stores multiple key-value pairs in a batch + */ + multiSet() { + return Promise.resolve(); + }, + + /** + * Merging an existing value with a new one + */ + mergeItem() { + return Promise.resolve(); + }, + + /** + * Multiple merging of existing and new values in a batch + * This function also removes all nested null values from an object. + */ + multiMerge() { + return Promise.resolve([]); + }, + + /** + * Remove given key and it's value from memory + */ + removeItem() { + return Promise.resolve(); + }, + + /** + * Remove given keys and their values from memory + */ + removeItems() { + return Promise.resolve(); + }, + + /** + * Clear everything from memory + */ + clear() { + return Promise.resolve(); + }, + + /** + * Returns all keys available in memory + */ + getAllKeys() { + return Promise.resolve([]); + }, + + /** + * Gets the total bytes of the store. + * `bytesRemaining` will always be `Number.POSITIVE_INFINITY` since we don't have a hard limit on memory. + */ + getDatabaseSize() { + return Promise.resolve({bytesRemaining: Number.POSITIVE_INFINITY, bytesUsed: 0}); + }, +}; + +export default provider; diff --git a/lib/storage/providers/SQLiteStorage.ts b/lib/storage/providers/SQLiteProvider.ts similarity index 83% rename from lib/storage/providers/SQLiteStorage.ts rename to lib/storage/providers/SQLiteProvider.ts index 36eb36227..ad5324ea9 100644 --- a/lib/storage/providers/SQLiteStorage.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -2,7 +2,7 @@ * The SQLiteStorage provider stores everything in a key/value store by * converting the value to a JSON string */ -import type {BatchQueryResult} from 'react-native-quick-sqlite'; +import type {BatchQueryResult, QuickSQLiteConnection} from 'react-native-quick-sqlite'; import {open} from 'react-native-quick-sqlite'; import {getFreeDiskStorage} from 'react-native-device-info'; import type StorageProvider from './types'; @@ -10,17 +10,27 @@ import utils from '../../utils'; import type {KeyList, KeyValuePairList} from './types'; const DB_NAME = 'OnyxDB'; -const db = open({name: DB_NAME}); +let db: QuickSQLiteConnection; -db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); +const provider: StorageProvider = { + /** + * The name of the provider that can be printed to the logs + */ + name: 'SQLiteProvider', + /** + * Initializes the storage provider + */ + init() { + db = open({name: DB_NAME}); -// All of the 3 pragmas below were suggested by SQLite team. -// You can find more info about them here: https://www.sqlite.org/pragma.html -db.execute('PRAGMA CACHE_SIZE=-20000;'); -db.execute('PRAGMA synchronous=NORMAL;'); -db.execute('PRAGMA journal_mode=WAL;'); + db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); -const provider: StorageProvider = { + // All of the 3 pragmas below were suggested by SQLite team. + // You can find more info about them here: https://www.sqlite.org/pragma.html + db.execute('PRAGMA CACHE_SIZE=-20000;'); + db.execute('PRAGMA synchronous=NORMAL;'); + db.execute('PRAGMA journal_mode=WAL;'); + }, getItem(key) { return db.executeAsync('SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key = ?;', [key]).then(({rows}) => { if (!rows || rows?.length === 0) { @@ -92,9 +102,6 @@ const provider: StorageProvider = { }; }); }, - - // eslint-disable-next-line @typescript-eslint/no-empty-function - keepInstancesSync: () => {}, }; export default provider; diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 353190f09..0472c4ff5 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -8,6 +8,14 @@ type KeyValuePairList = KeyValuePair[]; type OnStorageKeyChanged = (key: TKey, value: OnyxValue | null) => void; type StorageProvider = { + /** + * The name of the provider that can be printed to the logs + */ + name: string; + /** + * Initializes the storage provider + */ + init: () => void; /** * Gets the value of a given key or return `null` if it's not available in storage */ @@ -72,4 +80,4 @@ type StorageProvider = { }; export default StorageProvider; -export type {KeyList, KeyValuePair, KeyValuePairList}; +export type {KeyList, KeyValuePair, KeyValuePairList, OnStorageKeyChanged}; diff --git a/tests/unit/onyxMultiMergeWebStorageTest.js b/tests/unit/onyxMultiMergeWebStorageTest.js index 618b68d00..f1091bc6b 100644 --- a/tests/unit/onyxMultiMergeWebStorageTest.js +++ b/tests/unit/onyxMultiMergeWebStorageTest.js @@ -32,12 +32,12 @@ describe('Onyx.mergeCollection() and WebStorage', () => { afterEach(() => Onyx.clear()); it('merges two sets of data consecutively', () => { - StorageMock.setInitialMockData(initialData); + StorageMock.setMockStore(initialData); // Given initial data in storage - expect(StorageMock.getStorageMap().test_1).toEqual(initialTestObject); - expect(StorageMock.getStorageMap().test_2).toEqual(initialTestObject); - expect(StorageMock.getStorageMap().test_3).toEqual(initialTestObject); + expect(StorageMock.getMockStore().test_1).toEqual(initialTestObject); + expect(StorageMock.getMockStore().test_2).toEqual(initialTestObject); + expect(StorageMock.getMockStore().test_3).toEqual(initialTestObject); // And an empty cache values for the collection keys expect(OnyxCache.getValue('test_1')).not.toBeDefined(); @@ -75,17 +75,17 @@ describe('Onyx.mergeCollection() and WebStorage', () => { expect(OnyxCache.getValue('test_3')).toEqual(finalObject); // And the storage should reflect the same state - expect(StorageMock.getStorageMap().test_1).toEqual(finalObject); - expect(StorageMock.getStorageMap().test_2).toEqual(finalObject); - expect(StorageMock.getStorageMap().test_3).toEqual(finalObject); + expect(StorageMock.getMockStore().test_1).toEqual(finalObject); + expect(StorageMock.getMockStore().test_2).toEqual(finalObject); + expect(StorageMock.getMockStore().test_3).toEqual(finalObject); }); }); it('cache updates correctly when accessed again if keys are removed or evicted', () => { // Given empty storage - expect(StorageMock.getStorageMap().test_1).toBeFalsy(); - expect(StorageMock.getStorageMap().test_2).toBeFalsy(); - expect(StorageMock.getStorageMap().test_3).toBeFalsy(); + expect(StorageMock.getMockStore().test_1).toBeFalsy(); + expect(StorageMock.getMockStore().test_2).toBeFalsy(); + expect(StorageMock.getMockStore().test_3).toBeFalsy(); // And an empty cache values for the collection keys expect(OnyxCache.getValue('test_1')).toBeFalsy(); @@ -106,9 +106,9 @@ describe('Onyx.mergeCollection() and WebStorage', () => { expect(OnyxCache.getValue('test_1')).toEqual(data); expect(OnyxCache.getValue('test_2')).toEqual(data); expect(OnyxCache.getValue('test_3')).toEqual(data); - expect(StorageMock.getStorageMap().test_1).toEqual(data); - expect(StorageMock.getStorageMap().test_2).toEqual(data); - expect(StorageMock.getStorageMap().test_3).toEqual(data); + expect(StorageMock.getMockStore().test_1).toEqual(data); + expect(StorageMock.getMockStore().test_2).toEqual(data); + expect(StorageMock.getMockStore().test_3).toEqual(data); // When we drop all the cache keys (but do not modify the underlying storage) and merge another object OnyxCache.drop('test_1'); @@ -137,15 +137,15 @@ describe('Onyx.mergeCollection() and WebStorage', () => { expect(OnyxCache.getValue('test_3')).toEqual(finalObject); // And the storage should reflect the same state - expect(StorageMock.getStorageMap().test_1).toEqual(finalObject); - expect(StorageMock.getStorageMap().test_2).toEqual(finalObject); - expect(StorageMock.getStorageMap().test_3).toEqual(finalObject); + expect(StorageMock.getMockStore().test_1).toEqual(finalObject); + expect(StorageMock.getMockStore().test_2).toEqual(finalObject); + expect(StorageMock.getMockStore().test_3).toEqual(finalObject); }); }); it('setItem() and multiMerge()', () => { // Onyx should be empty after clear() is called - expect(StorageMock.getStorageMap()).toEqual({}); + expect(StorageMock.getMockStore()).toEqual({}); // Given no previous data and several calls to setItem and call to mergeCollection to update a given key @@ -174,7 +174,7 @@ describe('Onyx.mergeCollection() and WebStorage', () => { }; expect(OnyxCache.getValue('test_1')).toEqual(finalObject); - expect(StorageMock.getStorageMap().test_1).toEqual(finalObject); + expect(StorageMock.getMockStore().test_1).toEqual(finalObject); }); }); }); diff --git a/tests/unit/storage/providers/IDBKeyvalProviderTest.js b/tests/unit/storage/providers/IDBKeyvalProviderTest.js index ea4f5601a..c511cd0a3 100644 --- a/tests/unit/storage/providers/IDBKeyvalProviderTest.js +++ b/tests/unit/storage/providers/IDBKeyvalProviderTest.js @@ -1,6 +1,6 @@ import _ from 'underscore'; -import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyVal'; +import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyValProvider'; import createDeferredTask from '../../../../lib/createDeferredTask'; import waitForPromisesToResolve from '../../../utils/waitForPromisesToResolve'; @@ -64,7 +64,7 @@ describe('storage/providers/IDBKeyVal', () => { ]); return waitForPromisesToResolve().then(() => { - IDBKeyValProviderMock.idbKeyvalSet.mockClear(); + IDBKeyValProviderMock.mockSet.mockClear(); // Given deltas matching existing structure const USER_1_DELTA = { @@ -83,7 +83,7 @@ describe('storage/providers/IDBKeyVal', () => { ['@USER_2', USER_2_DELTA], ]).then(() => { // Then each existing item should be set with the merged content - expect(IDBKeyValProviderMock.idbKeyvalSet).toHaveBeenNthCalledWith(1, '@USER_1', { + expect(IDBKeyValProviderMock.mockSet).toHaveBeenNthCalledWith(1, '@USER_1', { name: 'Tom', age: 31, traits: { @@ -92,7 +92,7 @@ describe('storage/providers/IDBKeyVal', () => { }, }); - expect(IDBKeyValProviderMock.idbKeyvalSet).toHaveBeenNthCalledWith(2, '@USER_2', { + expect(IDBKeyValProviderMock.mockSet).toHaveBeenNthCalledWith(2, '@USER_2', { name: 'Sarah', age: 26, traits: { @@ -131,7 +131,7 @@ describe('storage/providers/IDBKeyVal', () => { // If StorageProvider.clear() does not abort the queue, more idbKeyval.setItem calls would be executed because they would // be sitting in the setItemQueue return waitForPromisesToResolve().then(() => { - expect(IDBKeyValProviderMock.idbKeyvalSet).toHaveBeenCalledTimes(0); + expect(IDBKeyValProviderMock.mockSet).toHaveBeenCalledTimes(0); expect(IDBKeyValProviderMock.clear).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/unit/storage/providers/StorageProviderTest.js b/tests/unit/storage/providers/StorageProviderTest.js index 82aca46b5..5ce43cfc9 100644 --- a/tests/unit/storage/providers/StorageProviderTest.js +++ b/tests/unit/storage/providers/StorageProviderTest.js @@ -1,11 +1,11 @@ /* eslint-disable import/first */ -jest.unmock('../../../../lib/storage/NativeStorage'); -jest.unmock('../../../../lib/storage/WebStorage'); -jest.unmock('../../../../lib/storage/providers/IDBKeyVal'); +jest.unmock('../../../../lib/storage/platforms/index.native'); +jest.unmock('../../../../lib/storage/platforms/index'); +jest.unmock('../../../../lib/storage/providers/IDBKeyValProvider'); import _ from 'underscore'; -import NativeStorage from '../../../../lib/storage/NativeStorage'; -import WebStorage from '../../../../lib/storage/WebStorage'; +import NativeStorage from '../../../../lib/storage/platforms/index.native'; +import WebStorage from '../../../../lib/storage/platforms/index'; it('storage providers have same methods implemented', () => { const nativeMethods = _.keys(NativeStorage);