From 177ae4ca746cb2454c6d1010621da9c7b5427acc Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Wed, 6 Mar 2024 17:48:33 +0100 Subject: [PATCH] Revert "Revert #475" This reverts commit 577e611d6b48c00abfe07e68837d373cf0a897f3. --- jestSetup.js | 6 +- lib/Onyx.js | 18 ++- lib/storage/InstanceSync/index.ts | 16 ++ lib/storage/InstanceSync/index.web.ts | 66 +++++++++ lib/storage/NativeStorage.ts | 3 - lib/storage/WebStorage.ts | 74 ---------- lib/storage/index.native.ts | 3 - lib/storage/index.ts | 138 +++++++++++++++++- lib/storage/platforms/index.native.ts | 3 + lib/storage/platforms/index.ts | 3 + .../{IDBKeyVal.ts => IDBKeyValProvider.ts} | 38 +++-- .../{SQLiteStorage.ts => SQLiteProvider.ts} | 26 ++-- lib/storage/providers/types.ts | 6 +- .../providers/IDBKeyvalProviderTest.js | 2 +- .../storage/providers/StorageProviderTest.js | 10 +- 15 files changed, 286 insertions(+), 126 deletions(-) create mode 100644 lib/storage/InstanceSync/index.ts create mode 100644 lib/storage/InstanceSync/index.web.ts delete mode 100644 lib/storage/NativeStorage.ts delete mode 100644 lib/storage/WebStorage.ts delete mode 100644 lib/storage/index.native.ts create mode 100644 lib/storage/platforms/index.native.ts create mode 100644 lib/storage/platforms/index.ts rename lib/storage/providers/{IDBKeyVal.ts => IDBKeyValProvider.ts} (70%) rename lib/storage/providers/{SQLiteStorage.ts => SQLiteProvider.ts} (84%) diff --git a/jestSetup.js b/jestSetup.js index 2288af999..0a5ef85fb 100644 --- a/jestSetup.js +++ b/jestSetup.js @@ -1,7 +1,7 @@ jest.mock('./lib/storage'); -jest.mock('./lib/storage/NativeStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/WebStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/providers/IDBKeyVal', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/index.native', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/index', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/providers/IDBKeyValProvider', () => require('./lib/storage/__mocks__')); jest.mock('react-native-device-info', () => ({getFreeDiskStorage: () => {}})); jest.mock('react-native-quick-sqlite', () => ({ diff --git a/lib/Onyx.js b/lib/Onyx.js index 8af2a87c7..bffb74072 100644 --- a/lib/Onyx.js +++ b/lib/Onyx.js @@ -1635,6 +1635,16 @@ function update(data) { * }); */ function init({keys = {}, initialKeyStates = {}, safeEvictionKeys = [], maxCachedKeysCount = 1000, shouldSyncMultipleInstances = Boolean(global.localStorage), debugSetState = false} = {}) { + Storage.init(); + + if (shouldSyncMultipleInstances) { + Storage.keepInstancesSync((key, value) => { + const prevValue = cache.getValue(key, false); + cache.set(key, value); + keyChanged(key, value, prevValue); + }); + } + if (debugSetState) { PerformanceUtils.setShouldDebugSetState(true); } @@ -1665,14 +1675,6 @@ function init({keys = {}, initialKeyStates = {}, safeEvictionKeys = [], maxCache // Initialize all of our keys with data provided then give green light to any pending connections Promise.all([addAllSafeEvictionKeysToRecentlyAccessedList(), initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); - - if (shouldSyncMultipleInstances && _.isFunction(Storage.keepInstancesSync)) { - Storage.keepInstancesSync((key, value) => { - const prevValue = cache.getValue(key, false); - cache.set(key, value); - keyChanged(key, value, prevValue); - }); - } } const Onyx = { diff --git a/lib/storage/InstanceSync/index.ts b/lib/storage/InstanceSync/index.ts new file mode 100644 index 000000000..327fbefdb --- /dev/null +++ b/lib/storage/InstanceSync/index.ts @@ -0,0 +1,16 @@ +import NOOP from 'lodash/noop'; + +/** + * This is used to keep multiple browser tabs in sync, therefore only needed on web + * On native platforms, we omit this syncing logic by setting this to mock implementation. + */ +const InstanceSync = { + init: NOOP, + setItem: NOOP, + removeItem: NOOP, + removeItems: NOOP, + mergeItem: NOOP, + clear: void>(callback: T) => Promise.resolve(callback()), +}; + +export default InstanceSync; diff --git a/lib/storage/InstanceSync/index.web.ts b/lib/storage/InstanceSync/index.web.ts new file mode 100644 index 000000000..a89cb54dc --- /dev/null +++ b/lib/storage/InstanceSync/index.web.ts @@ -0,0 +1,66 @@ +/* eslint-disable no-invalid-this */ +/** + * The InstancesSync object provides data-changed events like the ones that exist + * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when + * data changes and then stay up-to-date with everything happening in Onyx. + */ +import type {OnyxKey, OnyxValue} from '../../types'; +import type {KeyList, OnStorageKeyChanged} from '../providers/types'; + +const SYNC_ONYX = 'SYNC_ONYX'; + +/** + * Raise an event through `localStorage` to let other tabs know a value changed + * @param {String} onyxKey + */ +function raiseStorageSyncEvent(onyxKey: OnyxKey) { + global.localStorage.setItem(SYNC_ONYX, onyxKey); + global.localStorage.removeItem(SYNC_ONYX); +} + +function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { + onyxKeys.forEach((onyxKey) => { + raiseStorageSyncEvent(onyxKey); + }); +} + +const InstanceSync = { + /** + * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync + */ + init: (onStorageKeyChanged: OnStorageKeyChanged) => { + // This listener will only be triggered by events coming from other tabs + global.addEventListener('storage', (event) => { + // Ignore events that don't originate from the SYNC_ONYX logic + if (event.key !== SYNC_ONYX || !event.newValue) { + return; + } + + const onyxKey = event.newValue; + // @ts-expect-error `this` will be substituted later in actual function call + this.getItem(onyxKey).then((value: OnyxValue) => onStorageKeyChanged(onyxKey, value)); + }); + }, + setItem: raiseStorageSyncEvent, + removeItem: raiseStorageSyncEvent, + removeItems: raiseStorageSyncManyKeysEvent, + mergeItem: raiseStorageSyncEvent, + clear: (clearImplementation: () => void) => { + let allKeys: KeyList; + + // The keys must be retrieved before storage is cleared or else the list of keys would be empty + // @ts-expect-error `this` will be substituted later in actual function call + return this.getAllKeys() + .then((keys: KeyList) => { + allKeys = keys; + }) + .then(() => clearImplementation()) + .then(() => { + // Now that storage is cleared, the storage sync event can happen which is a more atomic action + // for other browser tabs + raiseStorageSyncManyKeysEvent(allKeys); + }); + }, +}; + +export default InstanceSync; diff --git a/lib/storage/NativeStorage.ts b/lib/storage/NativeStorage.ts deleted file mode 100644 index 1473613fa..000000000 --- a/lib/storage/NativeStorage.ts +++ /dev/null @@ -1,3 +0,0 @@ -import SQLiteStorage from './providers/SQLiteStorage'; - -export default SQLiteStorage; diff --git a/lib/storage/WebStorage.ts b/lib/storage/WebStorage.ts deleted file mode 100644 index 6621a084b..000000000 --- a/lib/storage/WebStorage.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * This file is here to wrap IDBKeyVal with a layer that provides data-changed events like the ones that exist - * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when - * data changes and then stay up-to-date with everything happening in Onyx. - */ -import type {OnyxKey} from '../types'; -import Storage from './providers/IDBKeyVal'; -import type {KeyList} from './providers/types'; -import type StorageProvider from './providers/types'; - -const SYNC_ONYX = 'SYNC_ONYX'; - -/** - * Raise an event thorough `localStorage` to let other tabs know a value changed - */ -function raiseStorageSyncEvent(onyxKey: OnyxKey) { - global.localStorage.setItem(SYNC_ONYX, onyxKey); - global.localStorage.removeItem(SYNC_ONYX); -} - -function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { - onyxKeys.forEach((onyxKey) => { - raiseStorageSyncEvent(onyxKey); - }); -} - -const webStorage: StorageProvider = { - ...Storage, - /** - * @param onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync - */ - keepInstancesSync(onStorageKeyChanged) { - // Override set, remove and clear to raise storage events that we intercept in other tabs - this.setItem = (key, value) => Storage.setItem(key, value).then(() => raiseStorageSyncEvent(key)); - - this.removeItem = (key) => Storage.removeItem(key).then(() => raiseStorageSyncEvent(key)); - - this.removeItems = (keys) => Storage.removeItems(keys).then(() => raiseStorageSyncManyKeysEvent(keys)); - - this.mergeItem = (key, batchedChanges, modifiedData) => Storage.mergeItem(key, batchedChanges, modifiedData).then(() => raiseStorageSyncEvent(key)); - - // If we just call Storage.clear other tabs will have no idea which keys were available previously - // so that they can call keysChanged for them. That's why we iterate over every key and raise a storage sync - // event for each one - this.clear = () => { - let allKeys: KeyList; - - // The keys must be retrieved before storage is cleared or else the list of keys would be empty - return Storage.getAllKeys() - .then((keys) => { - allKeys = keys; - }) - .then(() => Storage.clear()) - .then(() => { - // Now that storage is cleared, the storage sync event can happen which is a more atomic action - // for other browser tabs - allKeys.forEach(raiseStorageSyncEvent); - }); - }; - - // This listener will only be triggered by events coming from other tabs - global.addEventListener('storage', (event) => { - // Ignore events that don't originate from the SYNC_ONYX logic - if (event.key !== SYNC_ONYX || !event.newValue) { - return; - } - - const onyxKey = event.newValue; - Storage.getItem(onyxKey).then((value) => onStorageKeyChanged(onyxKey, value)); - }); - }, -}; - -export default webStorage; diff --git a/lib/storage/index.native.ts b/lib/storage/index.native.ts deleted file mode 100644 index 51b21ca5a..000000000 --- a/lib/storage/index.native.ts +++ /dev/null @@ -1,3 +0,0 @@ -import NativeStorage from './NativeStorage'; - -export default NativeStorage; diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 4ee520d20..a38348fc2 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -1,3 +1,137 @@ -import WebStorage from './WebStorage'; +import PlatformStorage from './platforms'; +import InstanceSync from './InstanceSync'; +import type StorageProvider from './providers/types'; -export default WebStorage; +const provider = PlatformStorage; +let shouldKeepInstancesSync = false; + +type Storage = { + getStorageProvider: () => StorageProvider; +} & StorageProvider; + +const Storage: Storage = { + /** + * Returns the storage provider currently in use + */ + getStorageProvider() { + return provider; + }, + + /** + * Initializes all providers in the list of storage providers + * and enables fallback providers if necessary + */ + init() { + provider.init(); + }, + + /** + * Get the value of a given key or return `null` if it's not available + */ + getItem: (key) => provider.getItem(key), + + /** + * Get multiple key-value pairs for the give array of keys in a batch + */ + multiGet: (keys) => provider.multiGet(keys), + + /** + * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string + */ + setItem: (key, value) => { + const promise = provider.setItem(key, value); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.setItem(key)); + } + + return promise; + }, + + /** + * Stores multiple key-value pairs in a batch + */ + multiSet: (pairs) => provider.multiSet(pairs), + + /** + * Merging an existing value with a new one + */ + mergeItem: (key, changes, modifiedData) => { + const promise = provider.mergeItem(key, changes, modifiedData); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.mergeItem(key)); + } + + return promise; + }, + + /** + * Multiple merging of existing and new values in a batch + * This function also removes all nested null values from an object. + */ + multiMerge: (pairs) => provider.multiMerge(pairs), + + /** + * Removes given key and its value + */ + removeItem: (key) => { + const promise = provider.removeItem(key); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItem(key)); + } + + return promise; + }, + + /** + * Remove given keys and their values + */ + removeItems: (keys) => { + const promise = provider.removeItems(keys); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItems(keys)); + } + + return promise; + }, + + /** + * Clears everything + */ + clear: () => { + if (shouldKeepInstancesSync) { + return InstanceSync.clear(() => provider.clear()); + } + + return provider.clear(); + }, + + // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438 + setMemoryOnlyKeys: () => provider.setMemoryOnlyKeys(), + + /** + * Returns all available keys + */ + getAllKeys: () => provider.getAllKeys(), + + /** + * Gets the total bytes of the store + */ + getDatabaseSize: () => provider.getDatabaseSize(), + + /** + * @param onStorageKeyChanged - Storage synchronization mechanism keeping all opened tabs in sync (web only) + */ + keepInstancesSync(onStorageKeyChanged) { + // If InstanceSync is null, it means we're on a native platform and we don't need to keep instances in sync + if (InstanceSync == null) return; + + shouldKeepInstancesSync = true; + InstanceSync.init(onStorageKeyChanged); + }, +}; + +export default Storage; diff --git a/lib/storage/platforms/index.native.ts b/lib/storage/platforms/index.native.ts new file mode 100644 index 000000000..95822c4a5 --- /dev/null +++ b/lib/storage/platforms/index.native.ts @@ -0,0 +1,3 @@ +import NativeStorage from '../providers/SQLiteProvider'; + +export default NativeStorage; diff --git a/lib/storage/platforms/index.ts b/lib/storage/platforms/index.ts new file mode 100644 index 000000000..0b95dc97d --- /dev/null +++ b/lib/storage/platforms/index.ts @@ -0,0 +1,3 @@ +import WebStorage from '../providers/IDBKeyValProvider'; + +export default WebStorage; diff --git a/lib/storage/providers/IDBKeyVal.ts b/lib/storage/providers/IDBKeyValProvider.ts similarity index 70% rename from lib/storage/providers/IDBKeyVal.ts rename to lib/storage/providers/IDBKeyValProvider.ts index 6312d0e3e..340e1ad04 100644 --- a/lib/storage/providers/IDBKeyVal.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -6,19 +6,23 @@ import type {OnyxKey, OnyxValue} from '../../types'; // We don't want to initialize the store while the JS bundle loads as idb-keyval will try to use global.indexedDB // which might not be available in certain environments that load the bundle (e.g. electron main process). -let customStoreInstance: UseStore; -function getCustomStore(): UseStore { - if (!customStoreInstance) { - customStoreInstance = createStore('OnyxDB', 'keyvaluepairs'); - } - return customStoreInstance; -} +let idbKeyValStore: UseStore; const provider: StorageProvider = { - setItem: (key, value) => set(key, value, getCustomStore()), - multiGet: (keysParam) => getMany(keysParam, getCustomStore()).then((values) => values.map((value, index) => [keysParam[index], value])), + /** + * Initializes the storage provider + */ + init() { + const newIdbKeyValStore = createStore('OnyxDB', 'keyvaluepairs'); + + if (newIdbKeyValStore == null) throw Error('IDBKeyVal store could not be created'); + + idbKeyValStore = newIdbKeyValStore; + }, + setItem: (key, value) => set(key, value, idbKeyValStore), + multiGet: (keysParam) => getMany(keysParam, idbKeyValStore).then((values) => values.map((value, index) => [keysParam[index], value])), multiMerge: (pairs) => - getCustomStore()('readwrite', (store) => { + idbKeyValStore('readwrite', (store) => { // Note: we are using the manual store transaction here, to fit the read and update // of the items in one transaction to achieve best performance. const getValues = Promise.all(pairs.map(([key]) => promisifyRequest>(store.get(key)))); @@ -36,15 +40,17 @@ const provider: StorageProvider = { // Since Onyx also merged the existing value with the changes, we can just set the value directly return provider.setItem(key, modifiedData); }, - multiSet: (pairs) => setMany(pairs, getCustomStore()), - clear: () => clear(getCustomStore()), - getAllKeys: () => keys(getCustomStore()), + multiSet: (pairs) => setMany(pairs, idbKeyValStore), + clear: () => clear(idbKeyValStore), + // eslint-disable-next-line @typescript-eslint/no-empty-function + setMemoryOnlyKeys: () => {}, + getAllKeys: () => keys(idbKeyValStore), getItem: (key) => - get(key, getCustomStore()) + get(key, idbKeyValStore) // idb-keyval returns undefined for missing items, but this needs to return null so that idb-keyval does the same thing as SQLiteStorage. .then((val) => (val === undefined ? null : val)), - removeItem: (key) => del(key, getCustomStore()), - removeItems: (keysParam) => delMany(keysParam, getCustomStore()), + removeItem: (key) => del(key, idbKeyValStore), + removeItems: (keysParam) => delMany(keysParam, idbKeyValStore), getDatabaseSize() { if (!window.navigator || !window.navigator.storage) { throw new Error('StorageManager browser API unavailable'); diff --git a/lib/storage/providers/SQLiteStorage.ts b/lib/storage/providers/SQLiteProvider.ts similarity index 84% rename from lib/storage/providers/SQLiteStorage.ts rename to lib/storage/providers/SQLiteProvider.ts index 36eb36227..4b93e821c 100644 --- a/lib/storage/providers/SQLiteStorage.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -2,7 +2,7 @@ * The SQLiteStorage provider stores everything in a key/value store by * converting the value to a JSON string */ -import type {BatchQueryResult} from 'react-native-quick-sqlite'; +import type {BatchQueryResult, QuickSQLiteConnection} from 'react-native-quick-sqlite'; import {open} from 'react-native-quick-sqlite'; import {getFreeDiskStorage} from 'react-native-device-info'; import type StorageProvider from './types'; @@ -10,17 +10,23 @@ import utils from '../../utils'; import type {KeyList, KeyValuePairList} from './types'; const DB_NAME = 'OnyxDB'; -const db = open({name: DB_NAME}); +let db: QuickSQLiteConnection; -db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); +const provider: StorageProvider = { + /** + * Initializes the storage provider + */ + init() { + db = open({name: DB_NAME}); -// All of the 3 pragmas below were suggested by SQLite team. -// You can find more info about them here: https://www.sqlite.org/pragma.html -db.execute('PRAGMA CACHE_SIZE=-20000;'); -db.execute('PRAGMA synchronous=NORMAL;'); -db.execute('PRAGMA journal_mode=WAL;'); + db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); -const provider: StorageProvider = { + // All of the 3 pragmas below were suggested by SQLite team. + // You can find more info about them here: https://www.sqlite.org/pragma.html + db.execute('PRAGMA CACHE_SIZE=-20000;'); + db.execute('PRAGMA synchronous=NORMAL;'); + db.execute('PRAGMA journal_mode=WAL;'); + }, getItem(key) { return db.executeAsync('SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key = ?;', [key]).then(({rows}) => { if (!rows || rows?.length === 0) { @@ -94,7 +100,7 @@ const provider: StorageProvider = { }, // eslint-disable-next-line @typescript-eslint/no-empty-function - keepInstancesSync: () => {}, + setMemoryOnlyKeys: () => {}, }; export default provider; diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 353190f09..801f2185f 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -8,6 +8,10 @@ type KeyValuePairList = KeyValuePair[]; type OnStorageKeyChanged = (key: TKey, value: OnyxValue | null) => void; type StorageProvider = { + /** + * Initializes the storage provider + */ + init: () => void; /** * Gets the value of a given key or return `null` if it's not available in storage */ @@ -72,4 +76,4 @@ type StorageProvider = { }; export default StorageProvider; -export type {KeyList, KeyValuePair, KeyValuePairList}; +export type {KeyList, KeyValuePair, KeyValuePairList, OnStorageKeyChanged}; diff --git a/tests/unit/storage/providers/IDBKeyvalProviderTest.js b/tests/unit/storage/providers/IDBKeyvalProviderTest.js index 50e053e5e..c511cd0a3 100644 --- a/tests/unit/storage/providers/IDBKeyvalProviderTest.js +++ b/tests/unit/storage/providers/IDBKeyvalProviderTest.js @@ -1,6 +1,6 @@ import _ from 'underscore'; -import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyVal'; +import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyValProvider'; import createDeferredTask from '../../../../lib/createDeferredTask'; import waitForPromisesToResolve from '../../../utils/waitForPromisesToResolve'; diff --git a/tests/unit/storage/providers/StorageProviderTest.js b/tests/unit/storage/providers/StorageProviderTest.js index 82aca46b5..5ce43cfc9 100644 --- a/tests/unit/storage/providers/StorageProviderTest.js +++ b/tests/unit/storage/providers/StorageProviderTest.js @@ -1,11 +1,11 @@ /* eslint-disable import/first */ -jest.unmock('../../../../lib/storage/NativeStorage'); -jest.unmock('../../../../lib/storage/WebStorage'); -jest.unmock('../../../../lib/storage/providers/IDBKeyVal'); +jest.unmock('../../../../lib/storage/platforms/index.native'); +jest.unmock('../../../../lib/storage/platforms/index'); +jest.unmock('../../../../lib/storage/providers/IDBKeyValProvider'); import _ from 'underscore'; -import NativeStorage from '../../../../lib/storage/NativeStorage'; -import WebStorage from '../../../../lib/storage/WebStorage'; +import NativeStorage from '../../../../lib/storage/platforms/index.native'; +import WebStorage from '../../../../lib/storage/platforms/index'; it('storage providers have same methods implemented', () => { const nativeMethods = _.keys(NativeStorage);