diff --git a/package-lock.json b/package-lock.json index d7132b6482..6c77ed01d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ambire-common", - "version": "2.102.3", + "version": "2.106.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ambire-common", - "version": "2.102.3", + "version": "2.106.0", "dependencies": { "@ambire/signature-validator": "^1.5.0", "@corpus-core/colibri-stateless": "^1.1.30", @@ -22,6 +22,7 @@ "ethers": "^6.8.0", "events": "^3.3.0", "hash-wasm": "^4.12.0", + "idb": "^8.0.3", "js-yaml": "^4.1.0", "scrypt-js": "^3.0.1", "tldts": "7.0.17", @@ -58,6 +59,7 @@ "eslint-plugin-prettier": "5.2.1", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", + "fake-indexeddb": "^6.2.5", "globals": "15.9.0", "hardhat": "2.24.1", "hardhat-gas-reporter": "1.0.10", @@ -8861,6 +8863,16 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/fake-indexeddb": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", + "integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -9911,6 +9923,12 @@ "node": ">=0.10.0" } }, + "node_modules/idb": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", + "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", + "license": "ISC" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", diff --git a/package.json b/package.json index 1ce1d67075..01c24e657c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "ethers": "^6.8.0", "events": "^3.3.0", "hash-wasm": "^4.12.0", + "idb": "^8.0.3", "js-yaml": "^4.1.0", "scrypt-js": "^3.0.1", "tldts": "7.0.17", @@ -77,6 +78,7 @@ "eslint-plugin-prettier": "5.2.1", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", + "fake-indexeddb": "^6.2.5", "globals": "15.9.0", "hardhat": "2.24.1", "hardhat-gas-reporter": "1.0.10", diff --git a/src/controllers/AGENTS.md b/src/controllers/AGENTS.md index 4f647e012b..76b7f6de11 100644 --- a/src/controllers/AGENTS.md +++ b/src/controllers/AGENTS.md @@ -86,7 +86,49 @@ Most controllers have `initialLoadPromise` that resolves when the controller fin ### Example: The networks controller that reads the network list from storage (which is async) exposes an `initialLoadPromise` that resolves when the networks are loaded. Then, the providers controller awaits the networks controller's `initialLoadPromise` in its own `initialLoadPromise` before initializing the providers, ensuring it has the network data available. The networks controller also awaits its own `initialLoadPromise` in its methods that read the network list, to ensure the data is loaded before accessing it. +## IndexedDB persistence + +> `src/services/storage/README.md` documents the runtime side of this layer — module map, +> startup order, invariants, and the cost of each operation. This section is the recipe for +> putting a new controller on IDB; read that one to understand what already runs. + +Most controllers persist through `StorageController` (`chrome.storage.local` / `AsyncStorage`), which reads and writes a whole key as one blob. Controllers whose data grows without bound (transaction history, caches) can instead persist row-by-row in IndexedDB. `ActivityController` is the reference implementation. + +IDB is **not available everywhere**. The extension/web background calls `openAmbireIdb()` and passes the connection down through `MainController`; on mobile it passes `undefined`. A controller therefore never branches on IDB availability itself — it picks one of two interchangeable backends in its constructor and uses it unconditionally: + +```ts +this.#persistence = idb + ? new ActivityIdbStorage(idb) + : new ActivityKeyValueStorage(storage, () => this.#accountsOps) +``` + +### Two different things are called "migration" + +- **Schema migration** — stores and indexes _inside_ IDB. Declared in `AMBIRE_IDB_SCHEMA` (`services/storage/idbSchema.ts`) and applied by `reconcileSchema()` / `applyMigrations()` in `idbDatabase.ts` during `onupgradeneeded`. `openAmbireIdb()` is awaited before any controller is constructed, so these always complete before the first read. +- **Data migration** — moving a controller's existing payload _out of_ key-value storage _into_ IDB, once. This is `ensureMigrated()` on the backend, and it runs at controller load time. + +### Adding IDB persistence to a controller + +1. Add the store to `AMBIRE_IDB_SCHEMA` and bump `dbVersion` by 1. `reconcileSchema()` creates the store and its indexes — never create them by hand in a handler. +2. Add an entry to `migrationHandlers` in `idbDatabase.ts` for the new version. A no-op is fine; handlers exist only to transform existing rows. The entry is mandatory so a version bump is always deliberate — a test enforces it. +3. Declare a backend interface with the data methods the controller actually calls, plus `ensureMigrated(getStoredData, removeStoredData)` where `TLegacy` is the shape currently held in key-value storage. See `IActivityOpsBackend` (`interfaces/activity.ts`) for the pattern. Keep the interface to what is used _polymorphically_: `isEmpty()` and `migrateFromStorage()` are how the IDB implementation decides whether to migrate, so declare them on that class only. Putting them on the shared interface forces the key-value class to carry dead stub methods it never uses. +4. Implement it twice — once on IDB, once on key-value. `ensureMigrated` on the IDB implementation must, in order: return early if the store is not empty; return early if the legacy payload has no meaningful data (a blank payload would make the store non-empty and permanently skip a later real migration); write the payload; only THEN call `removeStoredData`, so a failed removal still leaves the migrated data in place and doesn't lose it. The key-value implementation makes `ensureMigrated` an outright no-op, since its data already lives in its final location. +5. Call `ensureMigrated()` as the **first await** in the controller's `#load()`, before any read. Otherwise the controller can observe an empty store while the migration is still in flight. +6. **If the IDB backend loads only a subset at startup, audit every in-memory consumer.** This is the easiest way to introduce a silent bug. `ActivityController`'s startup read returns only the 20 most recent finalized ops per chain (plus all pending ones), which quietly weakened address-poisoning detection until `hasAccountOpsSentTo` was changed to expand the cache on demand. Anything that reasons over the _full_ history must either load it explicitly or read from a separate durable index. + +### Other things to know + +- **A `StorageController` migration can no longer reach a key that has been migrated into IDB.** `StorageController.get()`/`set()` await `#storageMigrationsPromise`, so storage migrations always finish before a controller reads — that part is safe, and it is why `#migrateNetworkIdToChainId` (which rewrites the network keys inside `accountsOps`) works correctly today. But once the IDB data migration has run, the legacy blob is a frozen copy that nothing reads. A _new_ storage migration written against `accountsOps` would rewrite that dead copy and silently have no effect. Transform the IDB rows with an `idbDatabase.ts` migration handler instead. +- Do not delete the legacy key as part of a data migration while the IDB path is still new. Keep it as a safety-net copy and record a completion flag (e.g. `activityIdbMigrated`) instead. The controller must _read_ from that copy when the migration fails, otherwise it shows an empty state for the whole session while the data sits intact one key away. The flag itself has no consumer today — `ActivityController` writes it (`#recordHistoryLivesInIdb`) because it can only be recorded while IDB works, and a session that cannot open IDB can no longer tell "never had transactions" from "history is in IDB and unreachable". There is deliberately no banner or error for that state: it is detected inside `#load()` on service-worker startup, where errors reach nothing because no window is open, and a banner for a condition the user cannot act on was judged not worth the noise. +- A retained legacy key is **frozen at migration time**, not a live mirror — writes go to IDB only from then on. Treat it as a floor for recovery, never as a source of truth, and decide up front when it gets deleted: keeping it forever means every migrated user carries the data twice (for `accountsOps` that is tens of MB) plus a stale artifact that reads like current data to the next person who finds it. +- **Known limitation of the `isEmpty()` migration guard.** It cannot tell "never migrated" apart from "migrated, then the store was wiped, then partially repopulated". If IDB is wiped while the app is running and _anything_ is written before the next restart, that single row makes `isEmpty()` false and the legacy copy is never read again — so whatever it held stays stranded. A wipe followed by a restart with no write in between _does_ recover, which is the path that matters in practice: the extension holds `unlimitedStorage`, so routine browser eviction is not a factor and a wipe means corruption or deliberate user action. Accepted deliberately rather than fixed — detecting _partial_ loss needs either a persisted row count (which legitimate decreases from account removal and cap eviction turn into false positives) or reading the whole legacy blob on every startup (which reintroduces exactly the cost IDB exists to remove). Revisit only if a controller keeps its legacy key permanently, in which case the `terminated()` callback in `idbDatabase.ts` is the right signal to re-run the migration on. +- Bulk writes must tolerate malformed legacy rows. A blob written by an older app version can be missing fields; drop those rows with a warning rather than throwing mid-batch, and keep the batch atomic. A partial commit makes `isEmpty()` return false and permanently disables the migration retry. +- `PhishingIdbStorage` is fully tested but its store is deliberately **not** in the manifest, and `PhishingController` still persists through key-value storage. It is a ready reference implementation, not a wired feature. Add the store to `AMBIRE_IDB_SCHEMA` in the same change that wires the controller — not before, because the required `dbVersion` bump cannot be rolled back and is not worth carrying for a store nothing reads. +- **A `dbVersion` bump is effectively one-way.** Once a user's database has been upgraded, a build pinned to the older version cannot open it — `openDB` rejects with `VersionError`, `openAmbireIdb()` fails, and every controller silently falls back to key-value storage. For `ActivityController` that means the retained legacy blob becomes the source of truth again, silently missing everything written since the migration. Treat version bumps as unrollbackable: ship one on its own, not bundled with unrelated risk, and only when something actually reads the new structure. +- `services/storage/idbIntegration.test.ts` contains a self-contained `DummyController` that is the canonical template for the whole wiring. + ## Other rules: + - Never use raw `setInterval`. Always use `RecurringTimeout` from `@common/utils/RecurringTimeout`. - Long-running background intervals must be declared in `ContinuousUpdatesController`, which orchestrates their lifecycle based on app state and controller events. If you need a new background loop, add it there and wire its start/stop/restart logic through the existing event subscriptions. - Never call `this.storage.set()` in parallel. Always await the previous call before making another one. diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index 7748d56c8b..6c40824d27 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -6,9 +6,11 @@ import { pickBetterPoisoningMatch, ScoredAddressPoisoningMatch } from '@/libs/transfer/address-poisoning' +import { AccountOpsPersistence } from '@/services/storage/accountOpsPersistence' +import { AmbireIdbDatabase } from '@/services/storage/idbDatabase' import { Account, AccountId, IAccountsController } from '../../interfaces/account' -import { IActivityController } from '../../interfaces/activity' +import { IActivityController, InternalAccountsOps } from '../../interfaces/activity' import { Banner } from '../../interfaces/banner' import { IEventEmitterRegistryController } from '../../interfaces/eventEmitter' import { Fetch } from '../../interfaces/fetch' @@ -87,11 +89,6 @@ export interface Filters { identifiedBy?: AccountOpIdentifiedBy } -export interface InternalAccountsOps { - // account => network => SubmittedAccountOp[] - [key: string]: { [key: string]: SubmittedAccountOp[] } -} - export interface ExternalAccountOps { [account: string]: { [network: string]: SubmittedAccountOpLike[] } } @@ -99,6 +96,7 @@ export interface ExternalAccountOps { // We are limiting items array to include no more than 1000 records, // as we trim out the oldest ones (in the beginning of the items array). // We do this to maintain optimal storage and performance. + const trim = (items: T[], maxSize = 1000): void => { if (items.length > maxSize) { // If the array size is greater than maxSize, remove the last (oldest) item @@ -247,6 +245,8 @@ export class ActivityController extends EventEmitter implements IActivityControl #fetch: Fetch + #persistence: AccountOpsPersistence + #initialLoadPromise?: Promise #accounts: IAccountsController @@ -322,11 +322,23 @@ export class ActivityController extends EventEmitter implements IActivityControl portfolio: IPortfolioController, safe: ISafeController, onContractsDeployed: (network: Network) => Promise, - eventEmitterRegistry?: IEventEmitterRegistryController + eventEmitterRegistry?: IEventEmitterRegistryController, + idb?: AmbireIdbDatabase ) { super(eventEmitterRegistry) this.#storage = storage this.#fetch = fetch + + // idb is provided only by web/extension environments — the background calls + // openAmbireIdb() and passes the result through MainController. On mobile, + // idb is undefined and the controller falls back to chrome.storage / AsyncStorage. + this.#persistence = new AccountOpsPersistence({ + storage, + idb, + getCache: () => this.#accountsOps, + onError: ({ message, error }) => this.emitError({ level: 'silent', message, error }) + }) + this.#callRelayer = callRelayer this.#accounts = accounts this.#selectedAccount = selectedAccount @@ -343,8 +355,11 @@ export class ActivityController extends EventEmitter implements IActivityControl async #load(): Promise { await this.#accounts.initialLoadPromise await this.#selectedAccount.initialLoadPromise + + // Persistence owns migration, backend fallback and the bounded startup read. It never + // rejects, so #load() cannot break the controller for the session. const [accountsOps, externalAccountOps, signedMessages, sentToHistory] = await Promise.all([ - this.#storage.get('accountsOps', {}), + this.#persistence.init(), this.#storage.get('externalAccountOps', {}), this.#storage.get('signedMessages', {}), this.#storage.get('sentToHistory', { domains: {}, recipients: {} }) @@ -356,6 +371,21 @@ export class ActivityController extends EventEmitter implements IActivityControl this.#sentToHistory = sentToHistory this.emitUpdate() + + // After the update on purpose: this warms the op counts and records the migration flag, + // and nothing on screen waits for either. It never rejects, so it cannot break #load(). + await this.#persistence.finalizeInit(accountsOps) + } + + /** + * Total transactions an account has ever made, as far as persistence knows. + * + * Synchronous because BannerController evaluates minTxnsTotal/maxTxnsTotal in a sync + * callback. Do NOT swap this for the in-memory group lengths: with a partially-loading + * backend those are just the startup window, so a heavy account would target wrongly. + */ + getTotalOpsCountForAccount(accountAddr: string): number { + return this.#persistence.getTotalOpsCount(accountAddr) } /** @@ -385,7 +415,14 @@ export class ActivityController extends EventEmitter implements IActivityControl addressPoisoningMatch: null } + // An empty accountId means "scan every account", so resolve the list first and + // expand all of them. Both answers below are derived from the entire history: + // whether the user has ever sent here, and whether the recipient mimics an + // address they used before. Judging either from the startup window alone would + // under-report — a lookalike of an older recipient would raise no warning. const accounts = accountId ? [accountId] : Object.keys(this.#accountsOps) + await this.#persistence.ensureFullHistory(accounts) + let found = false let lastTimestamp: number | null = null const normalizedToAddress = toAddress.toLowerCase() @@ -465,14 +502,31 @@ export class ActivityController extends EventEmitter implements IActivityControl ) const enabledNetworkChainIds = this.#networks.networks.map(({ chainId }) => String(chainId)) - const internalAccountOpsByChain = this.#accountsOps[filters.account] || {} + let internalAccountOpsByChain = this.#accountsOps[filters.account] || {} const externalAccountOpsByChain = this.#externalAccountOps[filters.account] || {} + + // Expand the filtered chain so pagination can page past the startup window. + // Persistence owns the markers and the merge; a failure there just leaves the window. + if (filters.chainId) { + await this.#persistence.ensureGroupLoaded(filters.account, filters.chainId) + internalAccountOpsByChain = this.#accountsOps[filters.account] || internalAccountOpsByChain + } + const internalAccountOpsEntriesOnEnabledNetworks = Object.entries( internalAccountOpsByChain ).filter(([chainId]) => enabledNetworkChainIds.includes(chainId)) const internalAccountOps = new Set( internalAccountOpsEntriesOnEnabledNetworks.flatMap(([, accountOps]) => accountOps) ) + + // Build a set of all txnIds from internal ops for dedup at the merge point. + // External ops whose txnId matches an internal op are filtered out here — they are + // duplicates that #removeExternalAccountOpsMatchingInternalOps missed because the + // internal op was outside the startup window when the scanner ran. + const internalTxnIds = new Set( + [...internalAccountOps].flatMap((op) => getInternalAccountOpTxnIds(op).map(normalizeTxnId)) + ) + const accountOpsEntriesOnEnabledNetworks = enabledNetworkChainIds .map( (chainId) => @@ -480,7 +534,9 @@ export class ActivityController extends EventEmitter implements IActivityControl chainId, [ ...(internalAccountOpsByChain[chainId] || []), - ...(externalAccountOpsByChain[chainId] || []) + ...(externalAccountOpsByChain[chainId] || []).filter( + (extOp) => !extOp.txnId || !internalTxnIds.has(normalizeTxnId(extOp.txnId)) + ) ] ] as const ) @@ -606,8 +662,11 @@ export class ActivityController extends EventEmitter implements IActivityControl await Promise.all(promises) } - private async persistAccountsOps() { - await this.#storage.set('accountsOps', this.#accountsOps) + /** + * Persist changed ops, sync filtered views, and emit an update. + */ + private async persistAccountsOps(changedOps: SubmittedAccountOp[]) { + await this.#persistence.updateOps(changedOps) await this.syncFilteredAccountsOps() this.emitUpdate() } @@ -720,6 +779,12 @@ export class ActivityController extends EventEmitter implements IActivityControl if (!this.#accountsOps[accountAddr][chainId.toString()]) this.#accountsOps[accountAddr][chainId.toString()] = [] + // Capture the oldest op's id before mutating — it is the one trim() will .pop() if the + // group is at capacity. On IDB the group usually starts at the startup window, so this + // rarely fires and eviction falls to putSingleOp's own MAX_IDB_GROUP_SIZE check. + const group = this.#accountsOps[accountAddr][chainId.toString()]! + const trimmedId = group.length >= 1000 ? group[group.length - 1]?.id : undefined + // newest SubmittedAccountOp goes first in the list this.#accountsOps[accountAddr]![chainId.toString()]!.unshift({ ...accountOp }) trim(this.#accountsOps[accountAddr][chainId.toString()]!) @@ -730,9 +795,22 @@ export class ActivityController extends EventEmitter implements IActivityControl await this.syncFilteredAccountsOps() - await this.#storage.set('accountsOps', this.#accountsOps) - await this.#storage.set('sentToHistory', this.#sentToHistory) this.emitUpdate() + + // Persistence LAST, per the storage rule in controllers/AGENTS.md — the key-value + // adapter rewrites the whole blob, so awaiting it earlier would block the UI on + // serializing the entire history. + // + // Do not "fix" this by persisting first: syncFilteredAccountsOps() above may expand this + // group from the backend before the op is written, and the merge inside + // AccountOpsPersistence is what keeps the memory-only op. + await this.#persistence.addOp(accountAddr, chainId, accountOp, trimmedId) + + // sentToHistory is a small durable index and always lives in key-value storage, + // never IDB. Persisting it here is what lets the recipient fast path in + // hasAccountOpsSentTo survive a service worker restart — without this the map + // is rebuilt empty on every wake-up and every recipient looks new again. + await this.#storage.set('sentToHistory', this.#sentToHistory) } #recordRecipient( @@ -913,6 +991,7 @@ export class ActivityController extends EventEmitter implements IActivityControl externalAccountOps.unshift(submittedAccountOpLike) trim(externalAccountOps) + // externalAccountOps: using chrome.storage.local only (not migrated to IDB yet) await this.#storage.set('externalAccountOps', this.#externalAccountOps) await this.syncFilteredAccountsOps() this.emitUpdate() @@ -951,7 +1030,7 @@ export class ActivityController extends EventEmitter implements IActivityControl */ async backfillAccountOpBalanceChangesAndPersist(accountOps: SubmittedAccountOp[]) { await Promise.all(accountOps.map((accOp) => this.backfillAccountOpBalanceChanges(accOp))) - await this.persistAccountsOps() + await this.persistAccountsOps(accountOps) } /** @@ -1120,7 +1199,7 @@ export class ActivityController extends EventEmitter implements IActivityControl ) ) ) - await this.persistAccountsOps() + await this.persistAccountsOps(balanceChangesTasks.map((t) => t.accountOp)) } /** @@ -1433,7 +1512,7 @@ export class ActivityController extends EventEmitter implements IActivityControl if (shouldEmitUpdate) { // remove duplicates if encountered during a race condition await this.#removeExternalAccountOpsMatchingInternalOps(updatedAccountsOps) - await this.persistAccountsOps() + await this.persistAccountsOps(updatedAccountsOps) } // record the balance changes but do not await them @@ -1521,12 +1600,21 @@ export class ActivityController extends EventEmitter implements IActivityControl delete this.#accountsOps[address] delete this.#signedMessages[address] + // Recipients are keyed per account, so they go with it. `domains` is + // deliberately left alone — it is a global index shared across accounts + // (see SentToHistory in ./types.ts). + delete this.#sentToHistory.recipients[address] await this.syncFilteredAccountsOps() await this.syncSignedMessages() - await this.#storage.set('accountsOps', this.#accountsOps) + // signedMessages and sentToHistory are always persisted to storage (not IDB). await this.#storage.set('signedMessages', this.#signedMessages) + await this.#storage.set('sentToHistory', this.#sentToHistory) + + // Also clears this account's expansion markers and cached op count, so a re-add + // re-reads from the backend instead of trusting a cache that no longer exists. + await this.#persistence.removeAccount(address) this.emitUpdate() } diff --git a/src/controllers/activity/activityIdbMigration.test.ts b/src/controllers/activity/activityIdbMigration.test.ts new file mode 100644 index 0000000000..05c7293c1e --- /dev/null +++ b/src/controllers/activity/activityIdbMigration.test.ts @@ -0,0 +1,782 @@ +/** + * ActivityController #load() — migration and startup-read behaviour. + * + * These cover the wiring rather than the storage primitives (those live in + * services/storage/activityIdb.test.ts): + * - the legacy blob migrates into IDB before the first read + * - the legacy key is kept as a safety-net copy and the completion flag recorded + * - a restart skips migration and reads IDB + * - IDB going missing after a completed migration degrades to the legacy blob + * quietly, without breaking the controller + * + * The controller dependencies are stubbed rather than built through + * makeMainController: #load() only touches storage, the persistence backend, and + * the two initialLoadPromise gates, so a full MainController would add seconds of + * RPC mocking without covering anything extra. + */ + +import 'fake-indexeddb/auto' + +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { beforeEach, describe, expect, jest, test } from '@jest/globals' + +import { IStorageController } from '../../interfaces/storage' +import { AccountOpStatus } from '../../libs/accountOp/types' +import { ActivityIdbStorage, STARTUP_RECENT_OPS_LIMIT } from '../../services/storage/activityIdb' +import { + AmbireIdbDatabase, + openAmbireIdb, + resetAmbireIdbForTesting +} from '../../services/storage/idbDatabase' +import { StorageController } from '../storage/storage' +import { ActivityController } from './activity' + +import { produceMemoryStore } from '../../../test/helpers' + +const ACC = '0xB674F3fd5F43464dB0448a57529eAF37F04cceA5' +const CHAIN_1 = 1n +const PROBE_ADDRESS = '0x0000000000000000000000000000000000000001' + +function makeOp(id: string, timestamp: number, status = AccountOpStatus.Success) { + return { + id, + accountAddr: ACC, + chainId: CHAIN_1, + calls: [] as { to: string; value: bigint; data: string }[], + gasFeePayment: null, + status, + timestamp, + identifiedBy: { type: 'Transaction', identifier: `0x${id}` } + } +} + +/** An op that sends to `to`, so it registers as a recipient in the history scan. */ +function makeOpTo(id: string, timestamp: number, to: string) { + return { + ...makeOp(id, timestamp), + calls: [{ to, value: 0n, data: '0x' }] + } +} + +/** A legacy accountsOps blob holding a single chain group for ACC. */ +function legacyBlob(ops: ReturnType[]) { + return { [ACC]: { '1': ops } } +} + +const alreadyLoaded = { initialLoadPromise: Promise.resolve() } as any + +// filterAccountsOps reads #networks.networks, so anything exercising the filtered +// views needs real-looking chain ids here. +const networksStub = { networks: [{ chainId: CHAIN_1 }, { chainId: 137n }] } as any + +function makeController(storage: IStorageController, idb?: AmbireIdbDatabase) { + return new ActivityController( + storage, + (() => {}) as any, + (() => {}) as any, + { ...alreadyLoaded, accounts: [{ addr: ACC }] } as any, // accounts + alreadyLoaded, // selectedAccount + {} as any, // providers + networksStub, // networks + {} as any, // portfolio + {} as any, // safe + async () => {}, + undefined, // eventEmitterRegistry + idb + ) +} + +/** + * Awaits the controller's private #initialLoadPromise. hasAccountOpsSentTo is the + * cheapest public method that gates on it — everything it does afterwards is + * in-memory, so it needs none of the stubbed dependencies. + */ +async function awaitLoad(controller: ActivityController) { + await controller.hasAccountOpsSentTo(PROBE_ADDRESS, ACC) +} + +/** + * 'activityIdbMigrated' is not part of the shared StorageProps schema (see the + * comment next to ActivityController#getActivityIdbMigrated) — reading and + * writing it from outside the controller needs the same narrow casts. + */ +function getActivityIdbMigrated(storageToRead: IStorageController): Promise { + return (storageToRead.get as (key: string, defaultValue: boolean) => Promise)( + 'activityIdbMigrated', + false + ) +} + +let db: AmbireIdbDatabase +let storage: IStorageController +let rawStore: ReturnType + +beforeEach(async () => { + resetAmbireIdbForTesting() + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + // checkQuota() reads navigator.storage — stub it to avoid a ReferenceError. + ;(global as any).navigator = {} + + db = await openAmbireIdb() + rawStore = produceMemoryStore() + storage = new StorageController(rawStore) +}) + +describe('ActivityController — IDB migration on load', () => { + test('migrates the legacy accountsOps blob into IDB before the first read', async () => { + await storage.set('accountsOps', legacyBlob([makeOp('legacy-1', 1000)]) as any) + + await awaitLoad(makeController(storage, db)) + + const rows = await new ActivityIdbStorage(db).getOpsForAccountAndChain(ACC, '1') + expect(rows).toHaveLength(1) + expect(rows?.[0]?.id).toBe('legacy-1') + }) + + test('keeps the legacy key as a safety-net copy and records the migrated flag', async () => { + // The legacy key is intentionally NOT removed for now — see #migrateOpsToIdb. + await storage.set('accountsOps', legacyBlob([makeOp('legacy-1', 1000)]) as any) + + await awaitLoad(makeController(storage, db)) + + expect(await storage.get('accountsOps', {})).not.toEqual({}) + expect(await getActivityIdbMigrated(storage)).toBe(true) + }) + + test('a restart skips migration and keeps reading IDB, ignoring later legacy writes', async () => { + await storage.set('accountsOps', legacyBlob([makeOp('migrated', 1000)]) as any) + await awaitLoad(makeController(storage, db)) + + // Something writes the legacy key again after the migration completed. IDB is + // non-empty now, so it must be ignored rather than re-imported. + await storage.set('accountsOps', legacyBlob([makeOp('stale', 9000)]) as any) + + await awaitLoad(makeController(storage, db)) + + const ids = (await new ActivityIdbStorage(db).getOpsForAccountAndChain(ACC, '1'))?.map( + (op) => op.id + ) + expect(ids).toEqual(['migrated']) + expect(ids).not.toContain('stale') + }) + + test('does nothing when there is no legacy data — no flag, no error', async () => { + const controller = makeController(storage, db) + await awaitLoad(controller) + + expect(await new ActivityIdbStorage(db).isEmpty()).toBe(true) + expect(await getActivityIdbMigrated(storage)).toBe(false) + expect(controller.emittedErrors).toHaveLength(0) + }) + + test('an unusable legacy op does not block the rest of the history', async () => { + await storage.set('accountsOps', { + [ACC]: { + '1': [ + makeOp('good-1', 1000), + // Row from an older app version with no timestamp + { id: 'broken', accountAddr: ACC, chainId: CHAIN_1, status: 'success' }, + makeOp('good-2', 3000) + ] + } + } as any) + + await awaitLoad(makeController(storage, db)) + + const ids = (await new ActivityIdbStorage(db).getOpsForAccountAndChain(ACC, '1'))?.map( + (op) => op.id + ) + expect(ids).toEqual(['good-2', 'good-1']) + // Migration completed despite the bad row, so the flag is recorded + expect(await getActivityIdbMigrated(storage)).toBe(true) + }) + + test('a failed migration keeps the legacy key so the next start can retry', async () => { + await storage.set('accountsOps', legacyBlob([makeOp('legacy-1', 1000)]) as any) + + // Break the legacy read so ensureMigrated rejects before writing anything + const failing: IStorageController = Object.create(storage) + failing.get = (async (key: string, defaultValue?: any) => { + if (key === 'accountsOps') throw new Error('storage read failed') + return (storage.get as any)(key, defaultValue) + }) as IStorageController['get'] + + const controller = makeController(failing, db) + await awaitLoad(controller) + + // Init still completed, the failure was reported, and nothing was migrated + expect(controller.emittedErrors.length).toBeGreaterThan(0) + expect(await new ActivityIdbStorage(db).isEmpty()).toBe(true) + expect(await storage.get('accountsOps', {})).not.toEqual({}) + expect(await getActivityIdbMigrated(storage)).toBe(false) + }) + + test('a failed migration still shows history, read from the retained legacy blob', async () => { + // The IDB write fails, so IDB is left empty. Reading the startup set from IDB + // would show an empty history for the whole session even though the legacy copy + // is intact — the fallback is what keeping that copy is for. + const RECIPIENT = '0xF0cD725D2195b1D3f4BD038c3786005B793237DB' + await storage.set('accountsOps', legacyBlob([makeOpTo('legacy-op', 1000, RECIPIENT)]) as any) + + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'migrateFromStorage') + .mockRejectedValue(new Error('idb write failed') as never) + + const controller = makeController(storage, db) + const result = await controller.hasAccountOpsSentTo(RECIPIENT, ACC) + + expect(result.found).toBe(true) + expect(await new ActivityIdbStorage(db).isEmpty()).toBe(true) + // Reported silently — the user still sees their history, so there is nothing + // for them to act on + expect(controller.emittedErrors.map((e) => e.level)).toContain('silent') + + spy.mockRestore() + }) + + test('a failed startup read leaves the controller usable rather than rejecting forever', async () => { + // #load() runs from the constructor and is assigned to #initialLoadPromise, + // which every public method awaits. If it rejects, the controller is bricked + // for the session and the rejection is unhandled. + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'loadStartupOps') + .mockRejectedValue(new Error('idb read failed') as never) + + const controller = makeController(storage, db) + + await expect(awaitLoad(controller)).resolves.toBeUndefined() + expect(controller.emittedErrors.length).toBeGreaterThan(0) + + spy.mockRestore() + }) + + test('init survives when both the migration and the fallback read fail', async () => { + // The legacy read itself is what broke, so the fallback throws too. Init must + // still complete rather than leaving the controller permanently unloaded. + const failing: IStorageController = Object.create(storage) + failing.get = (async (key: string, defaultValue?: any) => { + if (key === 'accountsOps') throw new Error('storage read failed') + return (storage.get as any)(key, defaultValue) + }) as IStorageController['get'] + + const controller = makeController(failing, db) + + await expect(awaitLoad(controller)).resolves.toBeUndefined() + expect(controller.emittedErrors.length).toBeGreaterThan(0) + }) +}) + +describe('ActivityController — key-value path (no IDB)', () => { + test('never migrates and never sets the flag', async () => { + await storage.set('accountsOps', legacyBlob([makeOp('kv-1', 1000)]) as any) + + const controller = makeController(storage, undefined) + await awaitLoad(controller) + + // The blob stays exactly where it is — it is already the source of truth here + expect(await storage.get('accountsOps', {})).not.toEqual({}) + expect(await getActivityIdbMigrated(storage)).toBe(false) + expect(controller.emittedErrors).toHaveLength(0) + }) + + test('stays usable and silent when IDB is missing after a migration already completed', async () => { + // Previous session migrated into IDB. This session failed to open IDB, so the + // history exists but is unreachable. There is deliberately no user-facing surfacing + // for this — what must hold is that the controller still loads and reads the + // retained legacy blob instead of throwing or starting a divergent one. + await (storage.set as (key: string, value: boolean) => Promise)( + 'activityIdbMigrated', + true + ) + await storage.set('accountsOps', legacyBlob([makeOp('kv-1', 1000)]) as any) + + const controller = makeController(storage, undefined) + await awaitLoad(controller) + + expect(controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id)).toEqual([ + 'kv-1' + ]) + expect(controller.emittedErrors).toHaveLength(0) + }) + + test('stays quiet when IDB is missing and no migration ever ran', async () => { + const controller = makeController(storage, undefined) + await awaitLoad(controller) + + expect(controller.emittedErrors).toHaveLength(0) + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Full-history expansion +// +// The IDB startup read only holds STARTUP_RECENT_OPS_LIMIT finalized ops per +// chain. hasAccountOpsSentTo answers "have I ever sent here" and computes the +// address-poisoning match, both of which need the whole history — so it expands +// the cache on demand first. +// ───────────────────────────────────────────────────────────────────────────── + +describe('ActivityController — full history expansion', () => { + const OLD_RECIPIENT = '0xF0cD725D2195b1D3f4BD038c3786005B793237DB' + + /** Seeds IDB with `count` ops; the OLDEST one sends to OLD_RECIPIENT. */ + async function seedBeyondStartupWindow(count: number) { + const ops = [ + makeOpTo('oldest', 1, OLD_RECIPIENT), + ...Array.from({ length: count - 1 }, (_, i) => makeOp(`recent-${i}`, 1000 + i)) + ] + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, ops as any) + } + + test('finds a recipient from an op older than the startup window', async () => { + // 25 ops, so the oldest falls outside the 20 finalized loaded at startup. + // Without expansion this reports found=false, and because the poisoning match + // is computed from the same scan, a lookalike of OLD_RECIPIENT would raise no + // warning on the send screen. + await seedBeyondStartupWindow(25) + + const controller = makeController(storage, db) + const result = await controller.hasAccountOpsSentTo(OLD_RECIPIENT, ACC) + + expect(result.found).toBe(true) + }) + + test('expands every scanned account when accountId is empty', async () => { + // An empty accountId means "scan all accounts". The expansion used to be keyed + // off that same empty argument, so it bailed immediately and the scan ran over + // the truncated startup window for every account. + await seedBeyondStartupWindow(25) + + const controller = makeController(storage, db) + const result = await controller.hasAccountOpsSentTo(OLD_RECIPIENT, '') + + expect(result.found).toBe(true) + }) + + test('expands each account only once across repeated calls', async () => { + await seedBeyondStartupWindow(25) + const spy = jest.spyOn(ActivityIdbStorage.prototype, 'getOpsForAccountAndChain') + + const controller = makeController(storage, db) + await controller.hasAccountOpsSentTo(OLD_RECIPIENT, ACC) + const afterFirst = spy.mock.calls.length + await controller.hasAccountOpsSentTo(OLD_RECIPIENT, ACC) + + // One fetch per chain on the first call, nothing on the second + expect(afterFirst).toBe(1) + expect(spy.mock.calls.length).toBe(afterFirst) + spy.mockRestore() + }) + + test('does not touch IDB on the key-value path', async () => { + await storage.set('accountsOps', legacyBlob([makeOpTo('kv', 1, OLD_RECIPIENT)]) as any) + const spy = jest.spyOn(ActivityIdbStorage.prototype, 'getOpsForAccountAndChain') + + const controller = makeController(storage, undefined) + const result = await controller.hasAccountOpsSentTo(OLD_RECIPIENT, ACC) + + // The key-value startup read already returns the full blob + expect(result.found).toBe(true) + expect(spy).not.toHaveBeenCalled() + spy.mockRestore() + }) + + test('a failed expansion is reported but still answers from the startup window', async () => { + await seedBeyondStartupWindow(25) + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'getOpsForAccountAndChain') + .mockRejectedValue(new Error('idb read failed') as never) + + const controller = makeController(storage, db) + const result = await controller.hasAccountOpsSentTo(OLD_RECIPIENT, ACC) + + // Degraded, not broken: the recent window is still searched + expect(result.found).toBe(false) + expect(controller.emittedErrors.length).toBeGreaterThan(0) + spy.mockRestore() + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Interactions between controller methods +// +// Each method below was already covered in isolation. These cover the SEQUENCES, +// which is where the real bugs were: a green suite of per-method tests missed all +// of them because none of them exercised two paths touching #accountsOps together. +// ───────────────────────────────────────────────────────────────────────────── + +describe('ActivityController — method interactions', () => { + const RECIPIENT = '0xF0cD725D2195b1D3f4BD038c3786005B793237DB' + + test('a new op is not dropped by a lazy-load triggered from the same call', async () => { + // Regression: with an active chain-filtered session and a group inside the startup + // window, addAccountOp's syncFilteredAccountsOps() lazy-loaded from IDB and + // REPLACED the in-memory group, discarding the op that had just been unshifted. + // The op reached disk but vanished from memory, so it was never polled to + // confirmation. + // + // This asserts the OUTCOME, not a mechanism, so it needs both defects present to + // fail: the per-group loaded flag (which stops the repeat lazy-load) and + // the persistence-layer merge (which keeps memory-only ops) each independently prevent it. + // Verified by restoring both the old length heuristic and replace-not-merge. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('existing', 1000) as any + ]) + + const controller = makeController(storage, db) + // Open a chain-filtered session, exactly as the history screen does + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + + await controller.addAccountOp(makeOp('brand-new', 9000) as any) + + const ids = controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id) + expect(ids).toContain('brand-new') + expect(ids).toContain('existing') + }) + + test('a failed migration keeps writes out of IDB so the guard can retry', async () => { + // Regression: on migration failure the session read the legacy blob but kept the + // IDB backend, so the first write put one row into the empty store. isEmpty() was + // then false forever and the real history was stranded permanently. + await storage.set('accountsOps', legacyBlob([makeOp('legacy-1', 1000)]) as any) + + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'migrateFromStorage') + .mockRejectedValue(new Error('idb write failed') as never) + + const controller = makeController(storage, db) + await awaitLoad(controller) + await controller.addAccountOp(makeOp('written-after-failure', 9000) as any) + + // IDB must still be empty, so the next startup retries the migration + expect(await new ActivityIdbStorage(db).isEmpty()).toBe(true) + // ...and the op went to the legacy blob instead, so it is not lost + const blob: any = await storage.get('accountsOps', {}) + const blobIds = Object.values(blob[ACC] ?? {}) + .flat() + .map((op: any) => op.id) + expect(blobIds).toContain('written-after-failure') + + spy.mockRestore() + }) + + test('pending ops pushing a group past the window do not block the lazy-load', async () => { + // Regression: the gate was `inMemoryCount > STARTUP_RECENT_OPS_LIMIT`. Pending ops + // are exempt from the 20-op cap, so 5 pending + 30 finalized arrives as 25 and the + // heuristic wrongly concluded the group was already fully expanded — leaving + // pagination showing 25 of 35. + const pending = Array.from({ length: 5 }, (_, i) => + makeOp(`pending-${i}`, 5000 + i, AccountOpStatus.BroadcastedButNotConfirmed) + ) + const finalized = Array.from({ length: 30 }, (_, i) => makeOp(`final-${i}`, 1000 + i)) + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + ...pending, + ...finalized + ] as any) + + const controller = makeController(storage, db) + await awaitLoad(controller) + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + + // All 35 must be reachable, not just the startup slice + const ids = controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id) + expect(ids).toHaveLength(35) + expect(ids).toContain('final-0') + }) + + test('an IDB user with no legacy blob still arms the stranded-history flag', async () => { + // Regression: ensureMigrated only set the flag after moving a legacy blob, which + // never happens for someone who installed after IDB became the default. The safety + // net therefore never armed for new users: a later IDB failure showed an empty + // history with no error at all. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('idb-native', 1000) as any + ]) + expect(await getActivityIdbMigrated(storage)).toBe(false) + + await awaitLoad(makeController(storage, db)) + + expect(await getActivityIdbMigrated(storage)).toBe(true) + }) + + test('a brand-new wallet with no ops does not arm the flag', async () => { + // The flag means "history lives in IDB". With no history there is nothing to warn + // about, so a fresh wallet must not be told it lost something. + await awaitLoad(makeController(storage, db)) + + expect(await getActivityIdbMigrated(storage)).toBe(false) + }) + + test('init survives the post-load history checks throwing', async () => { + // recording the migration flag was awaited unguarded at the end of #load, so a storage + // failure there rejected #initialLoadPromise for the whole session — exactly the + // failure mode guarded against 20 lines earlier in the same method. + // + // IDB must have ops for this to bite: the flag writer returns early on an + // empty store, so without seeding, the flag is never read and the test would pass + // whether or not the guard exists. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('some-op', 1000) as any + ]) + + const failing: IStorageController = Object.create(storage) + failing.get = (async (key: string, defaultValue?: any) => { + if (key === 'activityIdbMigrated') throw new Error('flag read failed') + return (storage.get as any)(key, defaultValue) + }) as IStorageController['get'] + + const controller = makeController(failing, db) + + await expect(awaitLoad(controller)).resolves.toBeUndefined() + expect(controller.emittedErrors.length).toBeGreaterThan(0) + }) + + test('a failed startup read does not permanently mark history as expanded', async () => { + // Regression: #ensureAccountHistoryLoaded marked an account fully-loaded whenever + // it had no chains in memory. After a failed startup read that is every account, + // so the poisoning scan silently had nothing to search for the rest of the session + // even though IDB held the full history. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOpTo('old-recipient', 1, RECIPIENT) as any + ]) + + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'loadStartupOps') + .mockRejectedValueOnce(new Error('idb read failed') as never) + + const controller = makeController(storage, db) + await awaitLoad(controller) + spy.mockRestore() + + // The startup read failed, so nothing is cached — but a later expansion must still + // be attempted rather than short-circuited by a stale "already loaded" marker. + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + const ids = controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id) + expect(ids).toContain('old-recipient') + }) +}) + +describe('ActivityController — merge-not-replace on lazy-load', () => { + /** + * Awaits #initialLoadPromise WITHOUT expanding history. awaitLoad() goes through + * hasAccountOpsSentTo, which expands every group and marks it fully loaded — that + * would stop filterAccountsOps from lazy-loading at all, so these tests would pass + * whether or not the merge works. findMessage only touches #signedMessages. + */ + const awaitLoadOnly = (controller: ActivityController) => controller.findMessage(ACC, () => true) + + test('an op that failed to persist still survives a later lazy-load', async () => { + // Isolates the persistence-layer merge. If persisting fails the op exists ONLY in memory, + // so a lazy-load that replaced the group with IDB content would erase it from the + // UI on top of having failed to save it. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('persisted', 1000) as any + ]) + + const spy = jest + .spyOn(ActivityIdbStorage.prototype, 'putSingleOp') + .mockRejectedValue(new Error('write failed') as never) + + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + await controller.addAccountOp(makeOp('memory-only', 9000) as any) + spy.mockRestore() + + // First lazy-load of this group, so the merge is what decides the outcome + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + + const ids = controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id) + expect(ids).toContain('memory-only') + expect(ids).toContain('persisted') + }) + + test('merging keeps in-memory object identity so in-flight mutations stick', async () => { + // updateAccountsOpsStatuses mutates op objects in place across long provider + // awaits. A concurrent lazy-load that swapped in fresh objects from IDB would send + // those mutations to detached copies, leaving the UI on stale state — so the + // cached object has to win on an id collision. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('shared', 1000, AccountOpStatus.BroadcastedButNotConfirmed) as any + ]) + + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + + const before = controller + .getAccountOpsForAccount({ accountAddr: ACC }) + .find((op) => op.id === 'shared')! + expect(before).toBeDefined() + // Mutate in place, exactly as the status poller does + before.status = AccountOpStatus.Success + + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + + const after = controller + .getAccountOpsForAccount({ accountAddr: ACC }) + .find((op) => op.id === 'shared')! + // Same object, so the mutation survived instead of being overwritten by the + // still-pending row IDB returned + expect(after).toBe(before) + expect(after.status).toBe(AccountOpStatus.Success) + }) +}) + +describe('ActivityController — total transaction count', () => { + // BannerController gates marketing banners on minTxnsTotal/maxTxnsTotal through a + // SYNCHRONOUS callback (see the AccountData callback in main.ts), so the count has to + // be cached. Using the in-memory group lengths instead reports the startup window and + // puts heavy accounts in the wrong targeting bucket. + const OVER_WINDOW = STARTUP_RECENT_OPS_LIMIT + 15 + + // hasAccountOpsSentTo (what awaitLoad uses) expands the full history as a side effect, + // which would hide the very gap these tests are about. findMessage only awaits the + // load promise. + const awaitLoadOnly = (controller: ActivityController) => controller.findMessage(ACC, () => true) + + test('reports the full persisted count, not the bounded startup window', async () => { + const ops = Array.from({ length: OVER_WINDOW }, (_, i) => makeOp(`op-${i}`, 1000 + i) as any) + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, ops) + + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + + // The startup read deliberately holds fewer than this in memory + expect(controller.getAccountOpsForAccount({ accountAddr: ACC }).length).toBeLessThan( + OVER_WINDOW + ) + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(OVER_WINDOW) + }) + + test('the op counts are warmed after the first update, not before it', async () => { + // finalizeInit() exists so counting (one backend query per account) cannot delay the + // first paint of the history. Folding it back into init() would reintroduce that. + const ops = Array.from({ length: OVER_WINDOW }, (_, i) => makeOp(`op-${i}`, 1000 + i) as any) + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, ops) + + const controller = makeController(storage, db) + const countsAtFirstUpdate: number[] = [] + controller.onUpdate(() => countsAtFirstUpdate.push(controller.getTotalOpsCountForAccount(ACC))) + + await awaitLoadOnly(controller) + + // The first update fires with the count still unwarmed (the in-memory lower bound) + expect(countsAtFirstUpdate[0]).toBeLessThan(OVER_WINDOW) + // ...and the warm count is available once load settles + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(OVER_WINDOW) + }) + + test('an account with no history reports zero', async () => { + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(0) + }) + + test('the count is exact on the key-value backend too', async () => { + await storage.set( + 'accountsOps', + legacyBlob([makeOp('kv-1', 1000), makeOp('kv-2', 2000)]) as any + ) + + const controller = makeController(storage, undefined) + await awaitLoadOnly(controller) + + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(2) + }) + + test('the mobile count reflects a newly added op with no refresh in between', async () => { + // On the key-value backend #accountsOps IS the whole history, so the count reads it + // live and the count refresh is skipped entirely — mobile does no extra work. + // + // NOTE: this asserts the observable guarantee, not the guard that provides it. The + // loadsPartially check in AccountOpsPersistence.getTotalOpsCount is defensive: because the refresh + // is gated too, the cache is always empty on this backend, so removing that check + // still leaves the test passing. It earns its place by keeping the property true if + // the refresh is ever un-gated. + await storage.set('accountsOps', legacyBlob([makeOp('kv-1', 1000)]) as any) + + const controller = makeController(storage, undefined) + await awaitLoadOnly(controller) + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(1) + + await controller.addAccountOp(makeOp('kv-2', 2000) as any) + + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(2) + }) + + test('removing an account drops its cached count', async () => { + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('gone', 1000) as any + ]) + + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(1) + + await controller.removeAccountData(ACC) + + // A stale cached count would keep reporting the removed account's transactions + expect(controller.getTotalOpsCountForAccount(ACC)).toBe(0) + }) +}) + +describe('ActivityController — bookkeeping around the expansion markers', () => { + const awaitLoadOnly = (controller: ActivityController) => controller.findMessage(ACC, () => true) + + test('removing an account clears its expansion markers so a re-add re-reads IDB', async () => { + // AccountOpsPersistence keys its expansion markers `${account}:${chainId}`, so removal has to clear by + // prefix. A stale marker would make a re-added account look already-expanded and + // permanently skip the lazy-load, showing only the startup window. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('first-life', 1000) as any + ]) + + const controller = makeController(storage, db) + // Expands chain 1 and marks it fully loaded + await controller.hasAccountOpsSentTo(PROBE_ADDRESS, ACC) + + await controller.removeAccountData(ACC) + + // The account comes back with fresh history in IDB + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('second-life', 2000) as any + ]) + await controller.filterAccountsOps('session-1', { account: ACC, chainId: CHAIN_1 }) + + const ids = controller.getAccountOpsForAccount({ accountAddr: ACC }).map((op) => op.id) + expect(ids).toContain('second-life') + }) + + test('a chain with no history is queried once, not on every update', async () => { + // getOpsForAccountAndChain returns undefined for zero rows. Marking the group + // loaded only on a non-empty result left every never-transacted-on chain unmarked, + // so it was re-queried on each filterAccountsOps call — which runs on every + // emitUpdate path. + await new ActivityIdbStorage(db).putOpsForAccountAndChain(ACC, CHAIN_1, [ + makeOp('on-chain-1', 1000) as any + ]) + + const controller = makeController(storage, db) + await awaitLoadOnly(controller) + + const spy = jest.spyOn(ActivityIdbStorage.prototype, 'getOpsForAccountAndChain') + + // Chain 137 is in the networks stub but the account has never used it + await controller.filterAccountsOps('session-empty', { account: ACC, chainId: 137n }) + await controller.filterAccountsOps('session-empty', { account: ACC, chainId: 137n }) + await controller.filterAccountsOps('session-empty', { account: ACC, chainId: 137n }) + + const emptyChainCalls = spy.mock.calls.filter(([, chainId]) => chainId === 137n) + expect(emptyChainCalls).toHaveLength(1) + + spy.mockRestore() + }) + + // NOTE: there is deliberately no test asserting that persistence happens before + // syncFilteredAccountsOps(). It must NOT — on the key-value backend putSingleOp + // rewrites the whole blob, so awaiting it before emitUpdate would block the UI on a + // full serialization of the history. the persistence-layer merge is what protects the new op, + // and 'a new op is not dropped by a lazy-load triggered from the same call' plus the + // merge tests cover that. +}) diff --git a/src/controllers/main/main.ts b/src/controllers/main/main.ts index edf6a87ebd..514c27feb9 100644 --- a/src/controllers/main/main.ts +++ b/src/controllers/main/main.ts @@ -74,6 +74,7 @@ import { IMainController, STATUS_WRAPPED_METHODS } from '@/interfaces/main' import { AddNetworkRequestParams, INetworksController, Network } from '@/interfaces/network' import { IPhishingController } from '@/interfaces/phishing' import { Platform } from '@/interfaces/platform' +import { AmbireIdbDatabase } from '@/services/storage/idbDatabase' import { IPortfolioController } from '@/interfaces/portfolio' import { IProvidersController } from '@/interfaces/provider' import { IRequestsController } from '@/interfaces/requests' @@ -233,7 +234,8 @@ export class MainController extends EventEmitter implements IMainController { featureFlags, keystoreSigners, externalSignerControllers, - uiManager + uiManager, + idb }: { eventEmitterRegistry?: IEventEmitterRegistryController appVersion: string @@ -250,6 +252,7 @@ export class MainController extends EventEmitter implements IMainController { keystoreSigners: Partial<{ [key in Key['type']]: KeystoreSignerType }> externalSignerControllers: ExternalSignerControllers uiManager: UiManager + idb?: AmbireIdbDatabase }) { super(eventEmitterRegistry) this.#storageAPI = storageAPI @@ -366,10 +369,12 @@ export class MainController extends EventEmitter implements IMainController { const currentSelectedAcc = this.selectedAccount.account if (!currentSelectedAcc) return { status: 'no-selected-account' } let totalUsdBalance = this.selectedAccount.portfolio.totalBalance - let numberOfTransactions = this.activity.getAccountOpsForAccount({ - accountAddr: currentSelectedAcc.addr, - sortAccOps: false - }).length + // Not getAccountOpsForAccount().length — that returns the in-memory cache, which + // on the IndexedDB backend holds only the bounded startup window, so a heavy + // account would report ~20 per chain and match the wrong minTxnsTotal bucket. + const numberOfTransactions = this.activity.getTotalOpsCountForAccount( + currentSelectedAcc.addr + ) const hasKeys = getAccountKeysCount({ accountAddr: currentSelectedAcc.addr, @@ -490,7 +495,8 @@ export class MainController extends EventEmitter implements IMainController { async (network: Network) => { await this.setContractsDeployedToTrueIfDeployed(network) }, - eventEmitterRegistry + eventEmitterRegistry, + idb ) this.transferScanner = new TransfersScannerController({ activity: this.activity, diff --git a/src/interfaces/activity.ts b/src/interfaces/activity.ts index d090771981..ba5e2af128 100644 --- a/src/interfaces/activity.ts +++ b/src/interfaces/activity.ts @@ -1,5 +1,87 @@ +import { SubmittedAccountOp, SubmittedAccountOpLike } from '../libs/accountOp/submittedAccountOp' import { ControllerInterface } from './controller' export type IActivityController = ControllerInterface< InstanceType > + +export interface InternalAccountsOps { + // account => network => SubmittedAccountOp[] + [key: string]: { [key: string]: SubmittedAccountOp[] } +} + +/** + * Persistence backend for account ops: ActivityIdbStorage (IndexedDB) or + * ActivityKeyValueStorage (chrome.storage.local). ActivityController always holds one, so + * it never branches on IDB availability. + */ +export interface IActivityOpsBackend { + /** + * Whether loadStartupOps() returns only a window rather than the whole history. + * + * The capability that drives every behavioural difference between adapters — expansion + * markers, cache merging and the total-op count all exist only when this is true. Callers + * branch on this, never on the concrete class. + */ + readonly loadsPartially: boolean + + /** + * One-time migration of the legacy blob into IDB. No-op on the key-value backend. + * Takes callbacks to stay decoupled from IStorageController. + * + * isEmpty()/migrateFromStorage() stay off this interface — nobody calls them + * polymorphically, and declaring them would force dead stubs onto the key-value class. + */ + ensureMigrated( + getStoredOps: () => Promise, + removeStoredOps: () => Promise + ): Promise + + /** IDB: pending ops + up to 20 finalized per (account, chainId). Key-value: everything. */ + loadStartupOps(): Promise + + /** Write one new op, and delete the op the in-memory trim evicted (if any). */ + putSingleOp( + accountAddr: string, + chainId: bigint | string, + op: SubmittedAccountOp, + trimmedId?: string + ): Promise + + /** Update existing rows in place (status, balance changes). */ + updateOps(ops: SubmittedAccountOp[]): Promise + + /** Full history for one (account, chainId) — the lazy-load behind pagination. */ + getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise + + /** Write ops for one (account, chainId) pair. */ + putOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise + + /** Batch write across multiple (account, chainId) records. */ + putMultiple( + records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise + + /** Delete every op for an account, across all chains. */ + deleteAccount(accountAddr: string): Promise + + /** + * Total persisted ops for an account. Needed because the IDB startup read is a bounded + * window, so in-memory lengths are not a total. + */ + countOpsForAccount(accountAddr: string): Promise +} + +/** @deprecated Use IActivityOpsBackend */ +export type IActivityIdbStorage = IActivityOpsBackend diff --git a/src/services/storage/README.md b/src/services/storage/README.md new file mode 100644 index 0000000000..9f6f663d7d --- /dev/null +++ b/src/services/storage/README.md @@ -0,0 +1,210 @@ +# IndexedDB persistence layer + +Row-level persistence for controller data that grows without bound. `ActivityController` +(transaction history) is the only consumer today. + +This file covers the **runtime picture**: what each module does, the order things happen in, +the invariants, and what each operation costs. For the step-by-step recipe to put a *new* +controller on IDB, see the "IndexedDB persistence" section of `src/controllers/AGENTS.md`. + +## Why it exists + +`accountsOps` used to live in one key-value blob. Every new transaction re-serialized the +entire history, and every service-worker wake-up read all of it back. For a heavy account +that is tens of MB per write. IDB replaces that with row-level writes and a **bounded** +startup read. + +## Modules + +| File | Responsibility | +|---|---| +| `idbSchema.ts` | Declarative manifest: stores, keyPaths, indexes, `dbVersion`. The single source of truth for *structure*. Read by `reconcileSchema()`; contains no logic. | +| `idbDatabase.ts` | Connection lifecycle (`openAmbireIdb()` singleton, `blocking`, `terminated`, invalidation) and upgrade orchestration (`reconcileSchema()`, `applyMigrations()`). | +| `accountOpsPersistence.ts` | **The coordinator `ActivityController` talks to.** Picks an adapter, runs the data migration, falls back on failure, and keeps the in-memory cache coherent with a partially-loaded backend. | +| `activityIdb.ts` | Two `IActivityOpsBackend` adapters: `ActivityIdbStorage` (rows) and `ActivityKeyValueStorage` (blob, used on mobile). | +| `phishingIdb.ts` | A second reference implementation. Fully tested, **not wired** — its store is deliberately absent from the manifest. | + +## Adapters, and adding a service + +`IActivityOpsBackend` is the adapter contract — one implementation per storage service. +`AccountOpsPersistence` selects one and exposes plain methods (`init`, `ensureFullHistory`, +`ensureGroupLoaded`, `addOp`, `updateOps`, `removeAccount`, `getTotalOpsCount`), so the +controller never branches on which backend it got. + +One capability drives every behavioural difference: + +```ts +readonly loadsPartially: boolean +``` + +`true` for IndexedDB, whose startup read is a window. `false` for key-value, which reads the +whole blob. Expansion markers, cache merging and the cached op total all exist only when it is +`true` — and callers test this flag, never the concrete class. + +Adding **expo-sqlite** on mobile therefore means: write an `IActivityOpsBackend` adapter with +`loadsPartially = true`, and select it in `#pickAdapter`. Nothing in `ActivityController` +changes, and nothing else in this layer does either. + +## Startup order + +The ordering here is load-bearing, not incidental. + +``` +background.ts + └─ await openAmbireIdb() ← schema migrations complete inside this await + ├─ reconcileSchema() creates any missing store/index, idempotent + └─ applyMigrations() transforms existing rows, per version + └─ new MainController({ idb }) nothing can read before the await resolves + └─ new ActivityController + └─ #load() + ├─ ensureMigrated() data migration: blob → rows, once + ├─ loadStartupOps() the bounded read + └─ emitUpdate() UI renders +``` + +`openAmbireIdb()` is awaited **before** any controller is constructed. That is the whole +guarantee that no controller can observe a half-migrated schema. If it throws, `idb` is +`undefined` and every controller silently uses its key-value backend. + +## Two different things called "migration" + +Keeping these apart avoids most of the confusion in this layer. + +| | Schema migration | Data migration | +|---|---|---| +| Moves | Stores and indexes *inside* IDB | A controller's payload *into* IDB | +| Declared in | `idbSchema.ts` | the backend's `ensureMigrated()` | +| Runs during | `onupgradeneeded` | controller `#load()` | +| Frequency | once per `dbVersion` bump | once, ever | + +## Structure is declarative + +`AMBIRE_IDB_SCHEMA` is the single source of truth for stores and indexes. `reconcileSchema()` +creates anything in the manifest that does not exist yet, so a purely additive change needs +only a manifest entry plus a `dbVersion` bump — never a hand-written create-store handler. + +It runs on **every** upgrade and is idempotent, which closes two gaps a per-version handler +leaves open: + +- a fresh install and an upgrading install end up on identical structure +- a new index reaches users who already have the store, not just fresh installs + +It only ever **adds**. Removing a store or index from the manifest does not remove it from +databases that already have it — that needs an explicit `deleteObjectStore`/`deleteIndex` in +the handler for the version that drops it. + +## Writing a migration handler + +Handlers live in `migrationHandlers` in `idbDatabase.ts`, keyed by the version they migrate +**to**. Upgrading v(n) → v(m) runs n+1..m in order, inside the single `onupgradeneeded` +transaction. They exist for **data** transformations — rewriting or backfilling rows. +Structure comes from `reconcileSchema()`, which runs first, so a handler can use stores and +indexes added by the same upgrade. + +1. **Use `tx` for everything.** Only the versionchange transaction is valid inside a handler; + opening a new one will not participate in the upgrade. +2. **Handlers are synchronous.** Chain off the read, never `await` it: + ```ts + store.getAll().then((rows) => rows.forEach((r) => store.put(migrate(r)))) + ``` + The versionchange transaction survives microtasks, so requests issued from a `.then()` + still land inside the upgrade. Awaiting a non-IDB promise lets it commit and the writes + vanish silently. See invariant 1 below — this is the single most dangerous rule here. +3. **Never remove a handler.** The chain must stay walkable from any prior version. +4. **Every version `1..dbVersion` needs an entry**, even a no-op, so a bump is always + deliberate. A test in `idbIntegration.test.ts` enforces this. +5. **A key already migrated into IDB is unreachable from a `StorageController` migration.** + Transform it with a handler here instead — the legacy blob is a frozen copy nothing reads. + +## Invariants + +Breaking any of these is a silent data bug, not a crash. The handler rules above are the +other three; these are the ones that bite outside a migration. + +1. **A `dbVersion` bump cannot be rolled back.** An older build cannot open an upgraded + database — `openDB` rejects with `VersionError` and every controller falls back to + key-value. Ship bumps alone, and only when something reads the new structure. +2. **Bulk writes are atomic and tolerate malformed rows.** A legacy blob can be missing + fields; those rows are dropped with a warning. A partial commit would make `isEmpty()` + false and permanently disable the migration retry. +3. **The startup read is a window, not the history.** Anything reasoning over the *whole* + history must expand first. This is the easiest way to introduce a silent bug here — see + the cost table below. +4. **Account addresses are case-sensitive keys.** Rows are keyed on the address exactly as + written, and an `IDBKeyRange` cannot match case-insensitively — unlike the in-memory + `getAccountOpsAccountKey()` helper, which exists precisely because addresses are not + always stored checksummed. A lookup with different casing than the stored row silently + returns nothing. Pre-existing rather than introduced here; noted so nobody assumes the + in-memory workaround extends to the storage layer. + +## The startup window, and who has to care + +`loadStartupOps()` returns, per (account, chain): **all pending ops** plus the **20 most +recent finalized** ones. So in-memory group lengths are *not* totals. + +Two mechanisms exist because of that: + +- expansion markers — a per-`(account, chain)` flag marking groups expanded to full history + this session. It must be an explicit flag: pending ops are exempt from the cap, so + a group can exceed 20 without having been expanded, and a length check would be wrong. +- the cache merge — expansion **merges** by id and keeps the *cached* object on a + collision. The cache can hold ops IDB does not have yet (a just-broadcast op is in memory + before `putSingleOp` writes it), and objects that in-flight work still mutates in place. + Replacing the array would drop the former and detach the latter. + +## Cost model + +| Operation | Cost | +|---|---| +| `loadStartupOps()` | 2 transactions. Key-only cursor enumerates groups, then per-group queries run in parallel. Bounded by group count, not history size. | +| `putSingleOp()` | 1 row write, plus one `count()` when the caller passed no `trimmedId` (the common case on IDB, since groups start at 20 and rarely hit the in-memory cap). | +| `getOpsForAccountAndChain()` | Full group read. Triggered by pagination past the window, once per group per session. | +| `countOpsForAccount()` | `count()` over a key range — served from the index without deserializing rows. | +| `hasAccountOpsSentTo()` (first-time recipient) | **Expensive.** Expands the account's *entire* history into memory and scans every op. See below. | + +### The one path that defeats the bounded read + +`hasAccountOpsSentTo()` answers two questions — "have I sent here before?" and "does this +recipient mimic one I used before?" (address poisoning). Both are properties of the *whole* +history, so on a miss it calls `ensureFullHistory()` and scans everything. With an +empty `accountId` it does this for **every** account. + +There is a fast path: `sentToHistory.recipients[accountId][address]`, a small durable map of +recipient → last-sent timestamp. When it hits, nothing is loaded. + +But that map is only populated by `addAccountOp`, so it covers sends made *since the feature +shipped*. **There is no backfill from existing history.** For a user with pre-existing +history the map starts empty, so the expensive path runs on sends to recipients they have +used before — not only genuinely new ones. Once expanded, the memory stays inflated for the +session. + +Backfilling `recipients` from full history once (at data-migration time) would let both +questions be answered from the small map and remove `ensureFullHistory()` from this +path. It is the highest-value optimization left in this layer, and is deliberately *not* part +of the initial IDB change: it alters security-relevant address-poisoning behaviour and +deserves its own review. + +## Connection can die mid-session + +The handle captured at construction is not permanently valid. + +- `blocking()` — another context wants to upgrade. We close and drop the cached promise. +- `terminated()` — the browser killed the connection. We drop the cached promise. +- `#openTx()` — catches `InvalidStateError` on a dead handle, invalidates the singleton, and + reopens once. Without this, every write after such a close would be lost while the + controller still believed IDB was available. + +The database itself survives all three, so a reopen recovers fully. + +## Testing + +`fake-indexeddb` backs the unit tests. It does **not** reproduce the versionchange commit +timing that invariant 1 is about — that was verified manually in both browsers. + +| Suite | Covers | +|---|---| +| `activityIdb.test.ts` | Storage primitives, atomicity, malformed rows, reconnect | +| `idbIntegration.test.ts` | End-to-end wiring via a self-contained `DummyController` — the canonical template | +| `activityIdbMigration.test.ts` | Controller `#load()`: migration, startup read, expansion, counts | +| `idbDatabase.test.ts` | Singleton, schema reconciliation, handler-chain consistency | +| `phishingIdb.test.ts` | The unwired reference backend | diff --git a/src/services/storage/accountOpsPersistence.ts b/src/services/storage/accountOpsPersistence.ts new file mode 100644 index 0000000000..d64b3979e8 --- /dev/null +++ b/src/services/storage/accountOpsPersistence.ts @@ -0,0 +1,395 @@ +import { IActivityOpsBackend, InternalAccountsOps } from '../../interfaces/activity' +import { IStorageController } from '../../interfaces/storage' +import { SubmittedAccountOp } from '../../libs/accountOp/submittedAccountOp' +import { ActivityIdbStorage, ActivityKeyValueStorage } from './activityIdb' +import { AmbireIdbDatabase } from './idbDatabase' + +export interface PersistenceError { + message: string + error: Error +} + +interface AccountOpsPersistenceParams { + storage: IStorageController + /** The connection opened at startup, or undefined where IDB does not exist (mobile). */ + idb?: AmbireIdbDatabase + /** + * The controller's live in-memory ops. Expansion writes merged groups back into it, and + * the key-value adapter serializes it on every write. + */ + getCache: () => InternalAccountsOps + /** Reported instead of thrown — every method here degrades rather than failing a caller. */ + onError: (e: PersistenceError) => void +} + +/** + * Owns everything about *where* account ops live, so ActivityController does not have to. + * + * Picks a storage adapter from what it is given, runs the one-time data migration, falls + * back when that fails, and keeps the in-memory cache coherent with a partially-loaded + * backend. The controller calls plain methods and never branches on the backend. + * + * Adding a service (e.g. expo-sqlite on mobile) means writing an IActivityOpsBackend + * adapter and selecting it in #pickAdapter — nothing else here or in the controller changes. + * + * No method rejects. This runs behind ActivityController's #initialLoadPromise, which every + * public method awaits, so a single failure escaping here would break the controller for the + * whole session. + */ +export class AccountOpsPersistence { + #adapter: IActivityOpsBackend + + #storage: IStorageController + + #getCache: () => InternalAccountsOps + + #onError: (e: PersistenceError) => void + + // (account, chainId) groups expanded to full history this session, keyed `${addr}:${chainId}`. + // Must be an explicit flag, not a length check: pending ops are exempt from the startup + // cap, so a group can exceed the window without having been expanded. + #fullyLoadedGroups = new Set() + + // Total op count per account, for callers that need a true total synchronously. Only used with + // a partially-loading adapter; otherwise the cache is the whole history and is summed live. + #totalOpsCount = new Map() + + constructor({ storage, idb, getCache, onError }: AccountOpsPersistenceParams) { + this.#storage = storage + this.#getCache = getCache + this.#onError = onError + this.#adapter = this.#pickAdapter(idb) + } + + #pickAdapter(idb?: AmbireIdbDatabase): IActivityOpsBackend { + if (idb) return new ActivityIdbStorage(idb) + + return new ActivityKeyValueStorage(this.#storage, this.#getCache) + } + + /** + * Migrate if needed, then return the dataset to start the session with. + * + * The migration must complete before the read, or the read would observe an empty store + * while the migration is still in flight. + * + * Deliberately does NOT do the post-load bookkeeping — see finalizeInit(). + */ + async init(): Promise { + const migrated = await this.#migrate() + + // A failed migration leaves the target empty while the retained legacy blob still holds + // everything, so this session behaves like a pre-migration one and reads AND writes the + // legacy key. Continuing to write to IDB would put a row into the empty store, making + // the isEmpty() guard skip the retry forever and stranding the real history. + if (!migrated) this.#fallBackToKeyValue() + + return this.#loadStartupOps() + } + + /** + * Bookkeeping that nothing renders: record the migration flag and warm the op counts. + * + * Split out of init() so the caller can emit its first update BEFORE this runs. Counting + * costs one backend query per account, and no UI waits on the result — folding it into + * init() would delay the first paint of the history for no benefit. + */ + async finalizeInit(ops: InternalAccountsOps): Promise { + await this.#recordMigrationCompleted(ops) + await this.#refreshAllCounts(ops) + } + + /** Whether the active adapter loads only a window at startup rather than everything. */ + get loadsPartially(): boolean { + return this.#adapter.loadsPartially + } + + /** + * Expand the given accounts from the startup window to their full history, for callers + * that must reason over every past op rather than the recent slice. + */ + async ensureFullHistory(accountAddrs: string[]): Promise { + if (!this.loadsPartially) return + + await Promise.all(accountAddrs.map((addr) => this.#expandAccount(addr))) + } + + /** + * Expand one (account, chain) group, for pagination past the startup window. + * + * On failure the group stays unmarked and the cache keeps the startup window — a subset + * rather than wrong data — so the caller can always page over whatever is there. + */ + async ensureGroupLoaded(accountAddr: string, chainId: bigint | string): Promise { + const chainIdStr = chainId.toString() + if (this.#isGroupLoaded(accountAddr, chainIdStr)) return + + try { + const fullOps = await this.#adapter.getOpsForAccountAndChain(accountAddr, chainId) + if (fullOps) this.#mergeIntoCache(accountAddr, chainIdStr, fullOps) + + // Marked even on an empty result: undefined means this group has no history to + // expand, not that expanding failed. Only marking on a hit would re-query on every + // call for any chain the account has never used. A real failure throws below. + this.#markGroupLoaded(accountAddr, chainIdStr) + } catch (error) { + this.#report('Older transactions could not be loaded.', error, 'expand a group') + } + } + + /** Persist one new op, and delete the op the caller's in-memory trim evicted. */ + async addOp( + accountAddr: string, + chainId: bigint | string, + op: SubmittedAccountOp, + trimmedId?: string + ): Promise { + try { + await this.#adapter.putSingleOp(accountAddr, chainId, op, trimmedId) + } catch (error) { + this.#report('Your latest transaction could not be saved to your history.', error, 'add op') + } + + // Recounted, not incremented: putSingleOp may have evicted a row, making this net-zero. + await this.#refreshCount(accountAddr) + } + + async updateOps(ops: SubmittedAccountOp[]): Promise { + try { + await this.#adapter.updateOps(ops) + } catch (error) { + this.#report('Some transaction updates could not be saved.', error, 'update ops') + } + } + + /** Drop an account's rows and every marker keyed to it. */ + async removeAccount(accountAddr: string): Promise { + // Cleared first so a failed delete cannot leave stale markers behind claiming the + // account's history is loaded and counted. + for (const key of this.#fullyLoadedGroups) { + if (key.startsWith(`${accountAddr}:`)) this.#fullyLoadedGroups.delete(key) + } + this.#totalOpsCount.delete(accountAddr) + + try { + await this.#adapter.deleteAccount(accountAddr) + } catch (error) { + this.#report( + "Some of the removed account's transaction history could not be deleted.", + error, + 'delete account' + ) + } + } + + /** + * Total transactions an account has ever made. Synchronous because the consumer + * (BannerController's txn thresholds) evaluates inside a sync callback. + */ + getTotalOpsCount(accountAddr: string): number { + // With a fully-loading adapter the cache IS the whole history, so a live sum is exact + // and free — a stored count could only ever be staler. + if (!this.loadsPartially) return this.#countInCache(accountAddr) + + return this.#totalOpsCount.get(accountAddr) ?? this.#countInCache(accountAddr) + } + + // ────────────────────────────────────────────────────────────────────────────── + // Internals + // ────────────────────────────────────────────────────────────────────────────── + + /** + * @returns false only if the migration failed, meaning the target is empty and must not + * be read from. True on success and where it is a no-op. + */ + async #migrate(): Promise { + try { + await this.#adapter.ensureMigrated( + () => this.#storage.get('accountsOps', {}), + // The legacy key is kept as a safety-net copy while IDB is still new; only the + // completion flag is recorded in place of removing it. + async () => this.#setMigratedFlag(true) + ) + + return true + } catch (error) { + // Non-fatal: the legacy key is intact, this session reads from it, and the next + // startup retries. The user sees their history either way. + this.#report( + 'Your transaction history could not be moved to its new location.', + error, + 'migrate to IDB' + ) + + return false + } + } + + #fallBackToKeyValue(): void { + if (!this.loadsPartially) return + + this.#adapter = new ActivityKeyValueStorage(this.#storage, this.#getCache) + } + + async #loadStartupOps(): Promise { + try { + return await this.#adapter.loadStartupOps() + } catch (error) { + // Degrading to empty keeps the controller usable; the data is untouched on disk and + // the next startup reads it again. + this.#report('Your transaction history could not be loaded.', error, 'read startup ops') + + return {} + } + } + + /** + * Record that this wallet's history lives in IDB. + * + * Nothing reads the flag yet. It is written anyway because it can only be recorded while + * IDB works — a session that cannot open IDB can no longer tell "never had transactions" + * from "history is in IDB and unreachable". + * + * A second writer is needed because ensureMigrated only sets it after moving a legacy + * blob, which never happens for users who installed after IDB became the default. Gated + * on there being ops, so a brand-new empty wallet is not marked as migrated. + */ + async #recordMigrationCompleted(ops: InternalAccountsOps): Promise { + if (!this.loadsPartially) return + if (!Object.keys(ops).length) return + + try { + if (await this.#getMigratedFlag()) return + await this.#setMigratedFlag(true) + } catch (error) { + this.#report('Your transaction history could not be checked.', error, 'record the flag') + } + } + + // 'activityIdbMigrated' is intentionally NOT part of the shared StorageProps schema + // (interfaces/storage.ts) — it is a provisional detail of the ongoing accountsOps → IDB + // migration. get()/set() are typed against StorageProps, so each needs one narrow cast. + #getMigratedFlag(): Promise { + return (this.#storage.get as (key: string, defaultValue: boolean) => Promise)( + 'activityIdbMigrated', + false + ) + } + + #setMigratedFlag(value: boolean): Promise { + return (this.#storage.set as (key: string, value: boolean) => Promise)( + 'activityIdbMigrated', + value + ) + } + + /** + * Expand every chain of one account. + * + * Only chains already in the cache are fetched, since loadStartupOps() enumerates every + * non-empty group. An account with no chains is deliberately NOT marked loaded — + * otherwise a failed startup read would convince us there is nothing to expand. + */ + async #expandAccount(accountAddr: string): Promise { + if (!accountAddr) return + + const chainIds = Object.keys(this.#getCache()[accountAddr] ?? {}).filter( + (chainId) => !this.#isGroupLoaded(accountAddr, chainId) + ) + if (!chainIds.length) return + + try { + const groups = await Promise.all( + chainIds.map(async (chainId) => ({ + chainId, + ops: await this.#adapter.getOpsForAccountAndChain(accountAddr, chainId) + })) + ) + + // Re-read: the account may have been removed while the reads were in flight. + if (!this.#getCache()[accountAddr]) return + + for (const { chainId, ops } of groups) { + if (ops?.length) this.#mergeIntoCache(accountAddr, chainId, ops) + this.#markGroupLoaded(accountAddr, chainId) + } + } catch (error) { + this.#report('Part of your transaction history could not be loaded.', error, 'expand account') + } + } + + /** + * Merge fetched rows into the cache, keeping the CACHED object on an id collision. + * + * A merge and not a replace, because the cache can hold ops the backend does not have yet + * (a just-broadcast op lands in memory before the write) and objects that in-flight work + * still mutates (status updates mutate across provider awaits). Replacing would drop the + * first and detach the second. + */ + #mergeIntoCache(accountAddr: string, chainId: string, fetched: SubmittedAccountOp[]): void { + const cache = this.#getCache() + if (!cache[accountAddr]) cache[accountAddr] = {} + + const cached = cache[accountAddr]![chainId] + if (!cached?.length) { + cache[accountAddr]![chainId] = [...fetched] + + return + } + + const byId = new Map() + for (const op of fetched) byId.set(op.id, op) + for (const op of cached) byId.set(op.id, op) + + cache[accountAddr]![chainId] = Array.from(byId.values()).sort( + (a, b) => b.timestamp - a.timestamp + ) + } + + #isGroupLoaded(accountAddr: string, chainId: string): boolean { + // A fully-loading adapter has everything already, so every group is loaded by definition + if (!this.loadsPartially) return true + + return this.#fullyLoadedGroups.has(`${accountAddr}:${chainId}`) + } + + #markGroupLoaded(accountAddr: string, chainId: string): void { + this.#fullyLoadedGroups.add(`${accountAddr}:${chainId}`) + } + + #countInCache(accountAddr: string): number { + return Object.values(this.#getCache()[accountAddr] ?? {}).reduce( + (total, ops) => total + (ops?.length ?? 0), + 0 + ) + } + + /** + * Only accounts present in the startup dataset are counted: loadStartupOps() enumerates + * every non-empty group, so an absent account has no ops and the cache sum of 0 is right. + */ + async #refreshAllCounts(ops: InternalAccountsOps): Promise { + if (!this.loadsPartially) return + + await Promise.all(Object.keys(ops).map((addr) => this.#refreshCount(addr))) + } + + async #refreshCount(accountAddr: string): Promise { + // Nothing to store when the cache is already the whole history. + if (!this.loadsPartially) return + + try { + this.#totalOpsCount.set(accountAddr, await this.#adapter.countOpsForAccount(accountAddr)) + } catch (error) { + // Leave the previous value; getTotalOpsCount falls back to the cache sum if unset. + this.#report('The transaction count could not be refreshed.', error, 'count ops') + } + } + + #report(message: string, error: unknown, what: string): void { + this.#onError({ + message, + error: error instanceof Error ? error : new Error(`AccountOpsPersistence: failed to ${what}`) + }) + } +} diff --git a/src/services/storage/activityIdb.test.ts b/src/services/storage/activityIdb.test.ts new file mode 100644 index 0000000000..72f2fcc6f1 --- /dev/null +++ b/src/services/storage/activityIdb.test.ts @@ -0,0 +1,1143 @@ +import 'fake-indexeddb/auto' + +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { beforeEach, describe, expect, jest, test } from '@jest/globals' + +import { SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' +import { AccountOpStatus } from '../../libs/accountOp/types' +import { ActivityIdbStorage, ActivityKeyValueStorage } from './activityIdb' +import { AmbireIdbDatabase, openAmbireIdb, resetAmbireIdbForTesting } from './idbDatabase' + +// ───────────────────────────────────────────────────────────────────────────── +// Test constants +// ───────────────────────────────────────────────────────────────────────────── + +const ACC_A = '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +const ACC_B = '0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB' +const CHAIN_1 = 1n +const CHAIN_137 = 137n + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +function makeOp( + id: string, + accountAddr: string, + chainId: bigint, + status: AccountOpStatus, + timestamp: number +): SubmittedAccountOpLike { + return { + id, + accountAddr, + chainId, + calls: [], + gasFeePayment: null as any, + status, + timestamp, + identifiedBy: { type: 'Transaction', identifier: `0x${id}` } + } as SubmittedAccountOpLike +} + +let db: AmbireIdbDatabase + +beforeEach(async () => { + // Reset the singleton and replace the in-memory IDB factory so each test + // gets a completely isolated environment. + resetAmbireIdbForTesting() + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + // checkQuota() reads navigator.storage — stub it to avoid ReferenceError in Node. + ;(global as any).navigator = {} + db = await openAmbireIdb() +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────── + +describe('ActivityIdbStorage', () => { + describe('isEmpty', () => { + test('returns true on a fresh store', async () => { + const store = new ActivityIdbStorage(db) + expect(await store.isEmpty()).toBe(true) + }) + + test('returns false after data is written', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + expect(await store.isEmpty()).toBe(false) + }) + }) + + describe('putOpsForAccountAndChain + getOpsForAccountAndChain', () => { + test('stores ops and returns them sorted by timestamp descending', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000), + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000), + makeOp('op-3', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result?.map((op) => op.id)).toEqual(['op-2', 'op-3', 'op-1']) + }) + + test('returns undefined when no ops exist for the pair', async () => { + const store = new ActivityIdbStorage(db) + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + }) + + test('accepts bigint chainId — retrieve with bigint or equivalent string', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_137, [ + makeOp('op-1', ACC_A, CHAIN_137, AccountOpStatus.Success, 1000) + ]) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, '137')).toHaveLength(1) + }) + + test('replaces existing ops on second write to the same pair', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('old', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('new-1', ACC_A, CHAIN_1, AccountOpStatus.Failure, 2000), + makeOp('new-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(2) + expect(result?.map((op) => op.id)).not.toContain('old') + }) + + test('different chains for the same account are stored independently', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('chain1-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_137, [ + makeOp('chain137-op', ACC_A, CHAIN_137, AccountOpStatus.Success, 2000) + ]) + + expect((await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))?.[0]?.id).toBe('chain1-op') + expect((await store.getOpsForAccountAndChain(ACC_A, CHAIN_137))?.[0]?.id).toBe('chain137-op') + }) + + test('concurrent writes from different accounts to the same chain do not interfere', async () => { + const store = new ActivityIdbStorage(db) + + await Promise.all([ + store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('a-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]), + store.putOpsForAccountAndChain(ACC_B, CHAIN_1, [ + makeOp('b-op', ACC_B, CHAIN_1, AccountOpStatus.Success, 2000) + ]) + ]) + + const aOps = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + const bOps = await store.getOpsForAccountAndChain(ACC_B, CHAIN_1) + + expect(aOps).toHaveLength(1) + expect(aOps?.[0]?.id).toBe('a-op') + expect(bOps).toHaveLength(1) + expect(bOps?.[0]?.id).toBe('b-op') + }) + + test('writing an empty ops array leaves the pair as if it never existed', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, []) + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + }) + + test('silently skips ops with no valid id', async () => { + const store = new ActivityIdbStorage(db) + const validOp = makeOp('valid', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + const invalidOp = { ...makeOp('', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) } + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [validOp as any, invalidOp as any]) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('valid') + }) + + test('deduplicates ops with the same id — last occurrence wins', async () => { + const store = new ActivityIdbStorage(db) + const v1 = makeOp('dup-id', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000) + const v2 = makeOp('dup-id', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [v1 as any, v2 as any]) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.status).toBe(AccountOpStatus.Success) + }) + }) + + describe('putMultiple', () => { + test('writes all (account, chainId) pairs atomically', async () => { + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b1', ACC_B, CHAIN_1, AccountOpStatus.Success, 3)] + } + ]) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_B, CHAIN_1)).toHaveLength(1) + }) + + test('replaces existing ops per pair', async () => { + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('old', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + } + ]) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('new', ACC_A, CHAIN_1, AccountOpStatus.Success, 2)] + } + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('new') + }) + + test('concurrent writes to the same (account, chain) pair — last writer wins', async () => { + const store = new ActivityIdbStorage(db) + + // IDB serializes readwrite transactions — the second one starts only after + // the first commits, so it deletes and rewrites with its own ops. + await Promise.all([ + store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('writer-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]), + store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('writer-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) + ]) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('writer-2') + }) + + test('duplicate (account, chain) pairs in one call — last record wins silently', async () => { + // #writeRecordToStore fires a range-delete + puts for each record in the + // same transaction. When two records share a pair, the second range-delete + // removes the first record's rows, leaving only the second record's ops. + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('first-pass', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('second-pass', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any] + } + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('second-pass') + }) + + test('a batch that fails mid-write commits nothing', async () => { + // Atomicity matters most during migration: a partial commit makes IDB + // non-empty, which permanently disables the ensureMigrated retry guard. + const store = new ActivityIdbStorage(db) + const good = makeOp('good', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + // Functions cannot pass through the structured clone algorithm + const unclonable = { + ...makeOp('unclonable', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000), + callback: () => {} + } + + await expect( + store.putMultiple([ + { accountAddr: ACC_A, chainId: CHAIN_1, ops: [good as any, unclonable as any] } + ]) + ).rejects.toThrow() + + expect(await store.isEmpty()).toBe(true) + }) + + test('an empty records array does not open a transaction', async () => { + // db is an idb Proxy, so jest.spyOn cannot instrument it — count through a + // thin wrapper that delegates to the real connection instead. + let transactionCalls = 0 + const countingDb = { + transaction: (...args: any[]) => { + transactionCalls += 1 + return (db as any).transaction(...args) + } + } + const store = new ActivityIdbStorage(countingDb as any) + + await store.putMultiple([]) + expect(transactionCalls).toBe(0) + + // A non-empty batch still opens exactly one + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('x', ACC_A, CHAIN_1, AccountOpStatus.Success, 1) as any] + } + ]) + expect(transactionCalls).toBe(1) + }) + }) + + describe('deleteAccount', () => { + test('removes all chains for the given account', async () => { + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + } + ]) + + await store.deleteAccount(ACC_A) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toBeUndefined() + }) + + test('does not affect other accounts', async () => { + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b', ACC_B, CHAIN_1, AccountOpStatus.Success, 2)] + } + ]) + + await store.deleteAccount(ACC_A) + + expect(await store.getOpsForAccountAndChain(ACC_B, CHAIN_1)).toHaveLength(1) + }) + + test('is a no-op when the account has no ops', async () => { + const store = new ActivityIdbStorage(db) + await expect(store.deleteAccount(ACC_A)).resolves.not.toThrow() + }) + + test('re-adding ops after deleteAccount works normally', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('original', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + await store.deleteAccount(ACC_A) + + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('readded', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('readded') + }) + }) + + describe('migrateFromStorage', () => { + test('imports all ops from InternalAccountsOps format', async () => { + const store = new ActivityIdbStorage(db) + await store.migrateFromStorage({ + [ACC_A]: { + '1': [makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any], + '137': [makeOp('op-2', ACC_A, CHAIN_137, AccountOpStatus.Failure, 2000) as any] + }, + [ACC_B]: { + '1': [makeOp('op-3', ACC_B, CHAIN_1, AccountOpStatus.Success, 3000) as any] + } + }) + + expect(await store.getOpsForAccountAndChain(ACC_A, '1')).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, '137')).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_B, '1')).toHaveLength(1) + }) + + test('preserves op ids and timestamps after migration', async () => { + const store = new ActivityIdbStorage(db) + await store.migrateFromStorage({ + [ACC_A]: { + '1': [makeOp('migrate-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 42000) as any] + } + }) + + const result = await store.getOpsForAccountAndChain(ACC_A, '1') + expect(result?.[0]?.id).toBe('migrate-op') + expect(result?.[0]?.timestamp).toBe(42000) + }) + + test('migrating an empty {} object writes nothing and leaves the store empty', async () => { + const store = new ActivityIdbStorage(db) + await store.migrateFromStorage({}) + expect(await store.isEmpty()).toBe(true) + }) + }) + + describe('ensureMigrated', () => { + test('migrates when IDB is empty and storage has ops', async () => { + const store = new ActivityIdbStorage(db) + const legacy = { + [ACC_A]: { + '1': [makeOp('legacy-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + } + } + + await store.ensureMigrated( + async () => legacy, + async () => {} + ) + + expect(await store.getOpsForAccountAndChain(ACC_A, '1')).toHaveLength(1) + }) + + test('calls removeStoredOps after a successful migration', async () => { + const store = new ActivityIdbStorage(db) + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated( + async () => ({ + [ACC_A]: { + '1': [makeOp('op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + } + }), + removeSpy + ) + + expect(removeSpy).toHaveBeenCalledTimes(1) + }) + + test('skips migration when IDB already has data', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('existing', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + + const getStoredSpy = jest.fn(async () => ({})) + await store.ensureMigrated(getStoredSpy, async () => {}) + + expect(getStoredSpy).not.toHaveBeenCalled() + }) + + test('a legacy op missing timestamp is dropped — the rest of the history still migrates', async () => { + // Regression: #opToRow used to throw mid-batch on such a row. The ops queued + // before it still committed, so IDB became non-empty, the isEmpty() guard + // skipped every future retry, and everything after the bad row was stranded + // in the legacy key forever. Unusable rows are now filtered out up front. + const store = new ActivityIdbStorage(db) + const legacy: any = { + [ACC_A]: { + '1': [ + makeOp('good-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000), + { id: 'no-timestamp', accountAddr: ACC_A, chainId: CHAIN_1, status: 'success' }, + makeOp('good-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000) + ] + } + } + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated(async () => legacy, removeSpy) + + const ids = (await store.getOpsForAccountAndChain(ACC_A, '1'))?.map((op) => op.id) + expect(ids).toEqual(['good-2', 'good-1']) + expect(ids).not.toContain('no-timestamp') + // Migration completed, so the legacy key is cleaned up rather than retried + expect(removeSpy).toHaveBeenCalledTimes(1) + }) + + test('a legacy op missing status is dropped without failing the batch', async () => { + const store = new ActivityIdbStorage(db) + const legacy: any = { + [ACC_A]: { + '1': [ + { id: 'no-status', accountAddr: ACC_A, chainId: CHAIN_1, timestamp: 500 }, + makeOp('good', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ] + } + } + + await store.ensureMigrated( + async () => legacy, + async () => {} + ) + + const ids = (await store.getOpsForAccountAndChain(ACC_A, '1'))?.map((op) => op.id) + expect(ids).toEqual(['good']) + }) + + test('concurrent ensureMigrated calls converge on the same migrated data', async () => { + const store1 = new ActivityIdbStorage(db) + const store2 = new ActivityIdbStorage(db) + const legacy = { + [ACC_A]: { + '1': [makeOp('concurrent', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + } + } + + // Both see isEmpty()=true before either writes; the second put is idempotent + await Promise.all([ + store1.ensureMigrated( + async () => legacy, + async () => {} + ), + store2.ensureMigrated( + async () => legacy, + async () => {} + ) + ]) + + const ops = await store1.getOpsForAccountAndChain(ACC_A, '1') + expect(ops).toHaveLength(1) + expect(ops?.[0]?.id).toBe('concurrent') + }) + + test('skips migration when storage returns an empty object', async () => { + const store = new ActivityIdbStorage(db) + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated(async () => ({}), removeSpy) + + expect(await store.isEmpty()).toBe(true) + expect(removeSpy).not.toHaveBeenCalled() + }) + + test('is idempotent — calling twice preserves the first migration result', async () => { + const store = new ActivityIdbStorage(db) + const legacy = { + [ACC_A]: { + '1': [makeOp('migrated', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + } + } + + await store.ensureMigrated( + async () => legacy, + async () => {} + ) + // Second call: IDB is non-empty — must not overwrite with stale data + await store.ensureMigrated( + async () => ({ + [ACC_A]: { '1': [makeOp('stale', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any] } + }), + async () => {} + ) + + const ops = await store.getOpsForAccountAndChain(ACC_A, '1') + expect(ops).toHaveLength(1) + expect(ops?.[0]?.id).toBe('migrated') + }) + + test('error from getStoredOps propagates and leaves IDB unchanged', async () => { + const store = new ActivityIdbStorage(db) + + await expect( + store.ensureMigrated( + async () => { + throw new Error('storage unavailable') + }, + async () => {} + ) + ).rejects.toThrow('storage unavailable') + + expect(await store.isEmpty()).toBe(true) + }) + + test('error from removeStoredOps propagates after IDB was already written', async () => { + const store = new ActivityIdbStorage(db) + const legacy = { + [ACC_A]: { + '1': [makeOp('legacy-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] + } + } + + await expect( + store.ensureMigrated( + async () => legacy, + async () => { + throw new Error('remove failed') + } + ) + ).rejects.toThrow('remove failed') + + // IDB was written before removeStoredOps was called + expect(await store.isEmpty()).toBe(false) + expect(await store.getOpsForAccountAndChain(ACC_A, '1')).toHaveLength(1) + }) + }) + + describe('loadStartupOps', () => { + test('returns an empty object for an empty store', async () => { + const store = new ActivityIdbStorage(db) + expect(await store.loadStartupOps()).toEqual({}) + }) + + test('returns all pending ops regardless of how many there are', async () => { + const store = new ActivityIdbStorage(db) + // 25 pending — more than the 20-op finalized limit + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(25) + }) + + test('limits finalized ops to 20 per (account, chainId) group', async () => { + const store = new ActivityIdbStorage(db) + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(20) + }) + + test('always includes pending ops even when the finalized limit is already reached', async () => { + const store = new ActivityIdbStorage(db) + const finalized = Array.from({ length: 20 }, (_, i) => + makeOp(`fin-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 10) + ) + const pending = Array.from({ length: 3 }, (_, i) => + makeOp(`pend-${i}`, ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000 + i) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [...finalized, ...pending]) + + const result = await store.loadStartupOps() + const ids = (result[ACC_A]?.['1'] ?? []).map((op) => op.id) + expect(ids.filter((id) => id.startsWith('pend-'))).toHaveLength(3) + expect(ids.filter((id) => id.startsWith('fin-'))).toHaveLength(20) + }) + + test('returns ops sorted by timestamp descending within each group', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 100), + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 300), + makeOp('op-3', ACC_A, CHAIN_1, AccountOpStatus.Success, 200) + ]) + + const result = await store.loadStartupOps() + expect((result[ACC_A]?.['1'] ?? []).map((op) => op.timestamp)).toEqual([300, 200, 100]) + }) + + test('handles multiple accounts and chains independently', async () => { + const store = new ActivityIdbStorage(db) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b1', ACC_B, CHAIN_1, AccountOpStatus.Success, 3)] + } + ]) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(1) + expect(result[ACC_A]?.['137']).toHaveLength(1) + expect(result[ACC_B]?.['1']).toHaveLength(1) + }) + + test('selects the 20 newest finalized ops (highest timestamps) per group', async () => { + const store = new ActivityIdbStorage(db) + // ops 0-24 — op with index 24 has the highest timestamp + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + const timestamps = (result[ACC_A]?.['1'] ?? []).map((op) => op.timestamp) + // Expect 2400, 2300, ..., 500 (the 20 newest) + expect(Math.min(...timestamps)).toBe(500) + expect(Math.max(...timestamps)).toBe(2400) + }) + + test('20-op finalized cap applies independently to each (account, chain) pair', async () => { + const store = new ActivityIdbStorage(db) + const make25 = (prefix: string, addr: string, chain: bigint) => + Array.from({ length: 25 }, (_, i) => + makeOp(`${prefix}-${i}`, addr, chain, AccountOpStatus.Success, i * 100) + ) + + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, make25('a1', ACC_A, CHAIN_1)) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_137, make25('a137', ACC_A, CHAIN_137)) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(20) + expect(result[ACC_A]?.['137']).toHaveLength(20) + }) + + test('treats AccountOpStatus.Pending as pending — not subject to the finalized cap', async () => { + const store = new ActivityIdbStorage(db) + const finalized = Array.from({ length: 20 }, (_, i) => + makeOp(`fin-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 10) + ) + // Queued ops (status=Pending) must also be returned uncapped, same as BroadcastedButNotConfirmed + const queued = Array.from({ length: 5 }, (_, i) => + makeOp(`queued-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Pending, 1000 + i) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [...finalized, ...queued] as any[]) + + const result = await store.loadStartupOps() + const ids = (result[ACC_A]?.['1'] ?? []).map((op) => op.id) + expect(ids.filter((id) => id.startsWith('queued-'))).toHaveLength(5) + expect(ids.filter((id) => id.startsWith('fin-'))).toHaveLength(20) + }) + + test('collects both BroadcastedButNotConfirmed and Pending via separate index queries', async () => { + // loadStartupOps fires two getAll() calls per group — one per pending status. + // A mix of both types in the same group verifies neither query is broken. + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('fin-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 100) as any, + makeOp('bcast-1', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 200) as any, + makeOp('bcast-2', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 300) as any, + makeOp('pend-1', ACC_A, CHAIN_1, AccountOpStatus.Pending, 400) as any, + makeOp('pend-2', ACC_A, CHAIN_1, AccountOpStatus.Pending, 500) as any + ] as any[]) + + const result = await store.loadStartupOps() + const ids = (result[ACC_A]?.['1'] ?? []).map((op) => op.id) + expect(ids).toHaveLength(5) + expect(ids).toContain('fin-1') + expect(ids).toContain('bcast-1') + expect(ids).toContain('bcast-2') + expect(ids).toContain('pend-1') + expect(ids).toContain('pend-2') + }) + }) + + describe('bigint serialization roundtrip', () => { + test('chainId and nonce survive serialize → store → deserialize', async () => { + const store = new ActivityIdbStorage(db) + const op = { + id: 'bigint-op', + accountAddr: ACC_A, + chainId: CHAIN_1, + nonce: 42n, + calls: [], + gasFeePayment: null, + status: AccountOpStatus.Success, + timestamp: 1000, + identifiedBy: { type: 'Transaction', identifier: '0xhash' } + } as unknown as SubmittedAccountOpLike + + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [op]) + const [retrieved] = (await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))! + + expect(retrieved?.chainId).toBe(CHAIN_1) + expect((retrieved as any).nonce).toBe(42n) + }) + }) + + describe('putSingleOp', () => { + test('adds a new op accessible via getOpsForAccountAndChain', async () => { + const store = new ActivityIdbStorage(db) + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('single-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('single-op') + }) + + test('evicts the op identified by trimmedId', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('old-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ]) + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('new-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any, + 'old-op' + ) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('new-op') + }) + + test('trimmedId that does not exist in IDB — no error, new op is still added', async () => { + const store = new ActivityIdbStorage(db) + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('added-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any, + 'ghost-id' + ) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('added-op') + }) + + test('coexists with ops written by putOpsForAccountAndChain', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('bulk-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ]) + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('single-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any + ) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(2) + expect(result!.map((op) => op.id)).toContain('bulk-op') + expect(result!.map((op) => op.id)).toContain('single-op') + }) + + test('throws when op has no valid id', async () => { + const store = new ActivityIdbStorage(db) + await expect( + store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ) + ).rejects.toThrow('Cannot store op without a valid id') + }) + + test('throws when op has no timestamp', async () => { + const store = new ActivityIdbStorage(db) + const invalid = { + ...makeOp('op-no-ts', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000), + timestamp: undefined + } + await expect(store.putSingleOp(ACC_A, CHAIN_1, invalid as any)).rejects.toThrow( + 'without a valid timestamp' + ) + }) + + test('evicts the oldest op when the group exceeds MAX_IDB_GROUP_SIZE (1000)', async () => { + const store = new ActivityIdbStorage(db) + // Fill to exactly 1000 ops (timestamps 0..999, oldest id is 'cap-op-0') + const ops = Array.from( + { length: 1000 }, + (_, i) => makeOp(`cap-op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i) as any + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + // Adding one more without trimmedId should trigger eviction of the oldest + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('cap-op-1000', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1000) + expect(result!.map((op) => op.id)).not.toContain('cap-op-0') + expect(result!.map((op) => op.id)).toContain('cap-op-1000') + }) + }) + + describe('updateOps', () => { + test('a malformed op aborts the batch, leaving earlier updates unapplied', async () => { + // Mirrors putMultiple's atomicity guarantee. #opToRow throws on an op with no + // timestamp; without tx.abort() the put already queued for the valid op would + // still commit, so callers would see a half-applied batch. + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-a', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000) as any + ]) + + const malformed = { + ...makeOp('op-b', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000), + timestamp: undefined + } + + await expect( + store.updateOps([ + makeOp('op-a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any, + malformed as any + ]) + ).rejects.toThrow('without a valid timestamp') + + // op-a must still be pending — the batch was rejected as a whole + const rows = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(rows).toHaveLength(1) + expect(rows?.[0]?.status).toBe(AccountOpStatus.BroadcastedButNotConfirmed) + }) + + test('updates the status of an existing op', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('update-me', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000) as any + ]) + + await store.updateOps([ + makeOp('update-me', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.status).toBe(AccountOpStatus.Success) + }) + + test('creates a new row when the op does not exist in IDB', async () => { + const store = new ActivityIdbStorage(db) + await store.updateOps([ + makeOp('new-via-update', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any + ]) + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result?.[0]?.id).toBe('new-via-update') + }) + + test('empty array is a no-op — store is unchanged', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('existing', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ]) + await store.updateOps([]) + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toHaveLength(1) + }) + + test('updates multiple ops in a single call', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-a', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000) as any, + makeOp('op-b', ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 2000) as any + ]) + + await store.updateOps([ + makeOp('op-a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any, + makeOp('op-b', ACC_A, CHAIN_1, AccountOpStatus.Failure, 2000) as any + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + const statusById = Object.fromEntries(result!.map((op) => [op.id, op.status])) + expect(statusById['op-a']).toBe(AccountOpStatus.Success) + expect(statusById['op-b']).toBe(AccountOpStatus.Failure) + }) + }) + + // The connection handle is captured at construction but can die later — + // blocking() closes it for an upgrade, and the browser can terminate it under + // storage pressure. The database survives, so a reopen recovers fully. + describe('recovery from a closed connection', () => { + // These deliberately use the PRODUCTION reconnect (the constructor default, + // openAmbireIdb) rather than a helper that resets the singleton first. An + // earlier version of these tests pre-reset it, which hid a real bug: on a close + // that fires neither blocking() nor terminated(), the cached promise still + // resolves to the dead connection, so recovery failed in production while the + // tests passed. + test('a write after the connection closes reopens and succeeds', async () => { + const store = new ActivityIdbStorage(db) + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('before', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ) + + db.close() + + await store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('after', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any + ) + + // Both rows are present, so the op written after the close was not lost + const ids = (await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))?.map((op) => op.id) + expect(ids).toEqual(['after', 'before']) + }) + + test('a read after the connection closes reopens and returns the data', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('persisted', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + + db.close() + + expect((await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))?.[0]?.id).toBe('persisted') + expect(await store.isEmpty()).toBe(false) + }) + + test('loadStartupOps recovers from a closed connection', async () => { + const store = new ActivityIdbStorage(db) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('startup', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + + db.close() + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(1) + }) + + test('errors unrelated to a dead connection are not retried', async () => { + // The only test here that injects a reconnect, so it can assert it is never + // reached. The rest deliberately use the production default. + const reconnect = jest.fn(openAmbireIdb) + const store = new ActivityIdbStorage(db, reconnect) + + // A validation failure must surface as-is rather than triggering a reopen + await expect( + store.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any + ) + ).rejects.toThrow('without a valid id') + + expect(reconnect).not.toHaveBeenCalled() + }) + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// ActivityKeyValueStorage +// ───────────────────────────────────────────────────────────────────────────── + +describe('ActivityKeyValueStorage', () => { + function makeStorageMock(initial: Record = {}) { + const store: Record = { ...initial } + return { + get: jest.fn(async (key: string, defaultValue: any) => + key in store ? structuredClone(store[key]) : defaultValue + ), + set: jest.fn(async (key: string, value: any) => { + store[key] = structuredClone(value) + }) + } + } + + test('loadStartupOps returns {} when storage is empty', async () => { + const backend = new ActivityKeyValueStorage(makeStorageMock() as any, () => ({})) + expect(await backend.loadStartupOps()).toEqual({}) + }) + + test('loadStartupOps returns the stored ops blob', async () => { + const stored = { + [ACC_A]: { '1': [makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] } + } + const backend = new ActivityKeyValueStorage( + makeStorageMock({ accountsOps: stored }) as any, + () => ({}) + ) + const result = await backend.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(1) + }) + + test('every write method persists the current getOps() snapshot to storage', async () => { + const inMemoryOps = { + [ACC_A]: { '1': [makeOp('op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any] } + } + const storage = makeStorageMock() + const backend = new ActivityKeyValueStorage(storage as any, () => inMemoryOps) + + await backend.putOpsForAccountAndChain(ACC_A, CHAIN_1, []) + expect(storage.set).toHaveBeenCalledWith('accountsOps', inMemoryOps) + storage.set.mockClear() + + await backend.putMultiple([]) + expect(storage.set).toHaveBeenCalledWith('accountsOps', inMemoryOps) + storage.set.mockClear() + + await backend.deleteAccount(ACC_A) + expect(storage.set).toHaveBeenCalledWith('accountsOps', inMemoryOps) + storage.set.mockClear() + + await backend.updateOps([]) + expect(storage.set).toHaveBeenCalledWith('accountsOps', inMemoryOps) + storage.set.mockClear() + + await backend.putSingleOp( + ACC_A, + CHAIN_1, + makeOp('new', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any + ) + expect(storage.set).toHaveBeenCalledWith('accountsOps', inMemoryOps) + }) + + test('getOpsForAccountAndChain reads in-memory state sorted by timestamp descending', async () => { + const inMemoryOps = { + [ACC_A]: { + '1': [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any, + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000) as any, + makeOp('op-3', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) as any + ] + } + } + const storage = makeStorageMock() + const backend = new ActivityKeyValueStorage(storage as any, () => inMemoryOps) + + const result = await backend.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result?.map((op) => op.id)).toEqual(['op-2', 'op-3', 'op-1']) + // Must read in-memory state — storage.get must not be called + expect(storage.get).not.toHaveBeenCalled() + }) + + test('getOpsForAccountAndChain returns undefined for an unknown account or chain', async () => { + const backend = new ActivityKeyValueStorage(makeStorageMock() as any, () => ({})) + expect(await backend.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + }) + + test('ensureMigrated is a no-op — neither callback is called', async () => { + const backend = new ActivityKeyValueStorage(makeStorageMock() as any, () => ({})) + const getOpsSpy = jest.fn(async () => ({})) + const removeSpy = jest.fn(async () => {}) + + await backend.ensureMigrated(getOpsSpy, removeSpy) + + expect(getOpsSpy).not.toHaveBeenCalled() + expect(removeSpy).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts new file mode 100644 index 0000000000..63e915f3cb --- /dev/null +++ b/src/services/storage/activityIdb.ts @@ -0,0 +1,609 @@ +import { IActivityOpsBackend, InternalAccountsOps } from '../../interfaces/activity' +import { IStorageController as IStorageControllerType } from '../../interfaces/storage' +import { SubmittedAccountOp, SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' +import { AccountOpStatus } from '../../libs/accountOp/types' +import { + AmbireIdbDatabase, + invalidateAmbireIdbConnection, + isClosedConnectionError, + openAmbireIdb +} from './idbDatabase' +import { AMBIRE_IDB_SCHEMA } from './idbSchema' + +/** + * Finalized ops loaded per (account, chainId) at startup. Pending ops are always + * loaded in full on top of this. + * + * Exported because ActivityController needs it to decide whether an in-memory + * group is still the startup window or has already been expanded from IDB. + */ +export const STARTUP_RECENT_OPS_LIMIT = 20 +// Hard cap on the number of IDB rows per (account, chainId) group. +// The in-memory cap is enforced by ActivityController; this guards against IDB +// accumulating more rows than the in-memory limit (e.g. after a startup that +// loaded only the 20-op subset before the limit was enforced). +const MAX_IDB_GROUP_SIZE = 1000 + +interface IdbAccountOpRow { + accountAddr: string + // String copy of op.chainId — BigInt is not a valid IDB key type so it cannot + // be used directly in the compound keyPath or index keys. + chainId: string + id: string + timestamp: number + status: AccountOpStatus + // The full op is stored via the Structured Clone Algorithm, which preserves + // BigInt natively — no JSON serialization needed. + op: SubmittedAccountOp | SubmittedAccountOpLike +} + +// The highest BMP Unicode character — used as a range upper bound to select all keys +// that start with a given prefix, without matching the prefix itself as a key. +const RANGE_HIGH = '￿' + +const STORE_DEF = AMBIRE_IDB_SCHEMA.stores.find((s) => s.storeName === 'accountsOps')! + +/** An op carrying every field the IDB row and its indexes require. */ +type StorableOp = (SubmittedAccountOp | SubmittedAccountOpLike) & { + id: string + timestamp: number + status: AccountOpStatus +} + +/** + * Bulk-write guard for ops from legacy blob storage, which may be missing timestamp or + * status. Such a row cannot be sorted or indexed, so bulk writes drop it with a warning — + * losing one unusable op beats failing the whole migration batch. + * + * putSingleOp does NOT use this: a live op missing these fields is a bug and must surface. + */ +function isStorableOp(op: SubmittedAccountOp | SubmittedAccountOpLike): op is StorableOp { + if (typeof op.id !== 'string' || !op.id) { + console.warn('[ActivityIdbStorage] Skipping op without a valid id', op) + return false + } + + if (typeof op.timestamp !== 'number') { + console.warn(`[ActivityIdbStorage] Skipping op ${op.id} without a valid timestamp`) + return false + } + + if (op.status === undefined) { + console.warn(`[ActivityIdbStorage] Skipping op ${op.id} without a valid status`) + return false + } + + return true +} + +export class ActivityIdbStorage implements IActivityOpsBackend { + // Startup reads STARTUP_RECENT_OPS_LIMIT finalized ops per group plus all pending ones. + readonly loadsPartially = true + + #db: AmbireIdbDatabase + + #storeName = STORE_DEF.storeName + + #reconnect: () => Promise + + /** + * @param db - The connection opened at startup. + * @param reconnect - Obtains a fresh connection when the current one dies. The + * default is the openAmbireIdb() singleton, which returns the + * cached connection normally and reopens once blocking() or + * terminated() has dropped it. Overridable for tests. + */ + constructor(db: AmbireIdbDatabase, reconnect: () => Promise = openAmbireIdb) { + this.#db = db + this.#reconnect = reconnect + } + + /** + * Open a transaction, reopening once if the handle turned out to be dead. + * + * The handle is captured at construction but can die later — blocking() closes it for an + * upgrade, and the browser can terminate it. The database survives, so a reopen recovers + * fully; without this, writes after such a close would be silently lost. + */ + // Generic over the mode so the transaction keeps its precise type — a widened union makes + // idb type the write methods as possibly-undefined. + async #openTx(mode: Mode) { + try { + return this.#db.transaction(this.#storeName, mode) + } catch (error) { + if (!isClosedConnectionError(error)) throw error + + console.warn('[ActivityIdbStorage] Connection was closed — reopening') + // Invalidate before reconnecting. blocking()/terminated() drop the cached + // connection when they fire, but a close can happen without either event — + // and then openAmbireIdb() would hand back the same dead handle and this + // retry would fail exactly like the original call. + invalidateAmbireIdbConnection() + this.#db = await this.#reconnect() + + return this.#db.transaction(this.#storeName, mode) + } + } + + // ────────────────────────────────────────────────────────────────────────────── + // Public API + // ────────────────────────────────────────────────────────────────────────────── + + /** + * Migrate the legacy blob into IDB, once. + * + * Emptiness is checked against IDB, not the legacy key, so a completed migration is cheap + * to skip and a wiped store recovers from the retained copy on the next restart. + * + * KNOWN LIMITATION: cannot tell "never migrated" from "wiped, then partially + * repopulated" — one row written after a wipe looks migrated. Accepted deliberately; see + * the IndexedDB section in src/controllers/AGENTS.md. + */ + async ensureMigrated( + getStoredOps: () => Promise, + removeStoredOps: () => Promise + ): Promise { + const empty = await this.isEmpty() + if (!empty) return + const storedOps = await getStoredOps() + if (Object.keys(storedOps).length === 0) return + await this.migrateFromStorage(storedOps) + await removeStoredOps() + } + + /** + * Load minimal startup dataset: all pending ops + up to STARTUP_RECENT_OPS_LIMIT + * finalized ops per (account, chain). + * + * Two transactions: a key-only cursor enumerates the (account, chainId) groups, then all + * per-group queries run in parallel inside one transaction — a timestamp cursor for the + * top N finalized, plus getAll on the status index for the pending ones. + * + * Every per-group request is fired before any await resolves, keeping the tx open. + */ + async loadStartupOps(): Promise { + // Step 1: enumerate (accountAddr, chainId) groups — key-only cursor, O(N_groups) reads + const groups: [string, string][] = [] + { + const tx = await this.#openTx('readonly') + let cursor = await tx.objectStore(this.#storeName).openKeyCursor() + while (cursor) { + const [accountAddr, chainId] = cursor.primaryKey as [string, string, string] + groups.push([accountAddr, chainId]) + cursor = await cursor.continue([accountAddr, chainId, RANGE_HIGH]) + } + } + + if (groups.length === 0) return {} + + // Step 2: fetch per-group data — all groups run in parallel within one transaction. + // Each group's async function fires its IDB requests (1 cursor + 2 getAlls) before + // the first await resolves, so the transaction always has pending requests. + const result: InternalAccountsOps = {} + { + const tx = await this.#openTx('readonly') + const store = tx.objectStore(this.#storeName) + const tsIndex = store.index('by-account-chain-timestamp') + const statusIndex = store.index('by-account-chain-status') + + await Promise.all( + groups.map(async ([accountAddr, chainId]) => { + if (!result[accountAddr]) result[accountAddr] = {} + if (!result[accountAddr]![chainId]) result[accountAddr]![chainId] = [] + const groupOps = result[accountAddr]![chainId]! + + const tsRange = IDBKeyRange.bound( + [accountAddr, chainId, 0], + [accountAddr, chainId, Number.MAX_SAFE_INTEGER] + ) + + // Run timestamp cursor + 2 pending getAlls in parallel for this group. + // The getAlls are fired synchronously (before any await), the cursor IIFE + // fires its first request synchronously too — all 3 are pending at once. + const [, pendingBroadcasted, pendingQueued] = await Promise.all([ + (async () => { + let finalizedCount = 0 + let cur = await tsIndex.openCursor(tsRange, 'prev') + while (cur && finalizedCount < STARTUP_RECENT_OPS_LIMIT) { + const row = cur.value as IdbAccountOpRow + const isPending = + row.status === AccountOpStatus.BroadcastedButNotConfirmed || + row.status === AccountOpStatus.Pending + if (!isPending) { + groupOps.push(row.op as SubmittedAccountOp) + finalizedCount++ + } + cur = await cur.continue() + } + })(), + statusIndex.getAll( + IDBKeyRange.only([accountAddr, chainId, AccountOpStatus.BroadcastedButNotConfirmed]) + ), + statusIndex.getAll(IDBKeyRange.only([accountAddr, chainId, AccountOpStatus.Pending])) + ]) + + for (const row of [...pendingBroadcasted, ...pendingQueued] as IdbAccountOpRow[]) { + groupOps.push(row.op as SubmittedAccountOp) + } + }) + ) + } + + // Sort each group descending by timestamp + for (const chainMap of Object.values(result)) { + for (const ops of Object.values(chainMap)) { + ops.sort((a, b) => b.timestamp - a.timestamp) + } + } + + return result + } + + /** + * Write a single new op and optionally delete the op evicted by the in-memory trim. + * O(1) IDB operations vs. the full-group rewrite of putOpsForAccountAndChain. + */ + async putSingleOp( + accountAddr: string, + chainId: bigint | string, + op: SubmittedAccountOp, + trimmedId?: string + ): Promise { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + const tx = await this.#openTx('readwrite') + const store = tx.objectStore(this.#storeName) + + // Fire put before any await so the transaction has a pending request. + // .catch(() => {}) suppresses unhandled-rejection warnings; tx.done still + // rejects on failure and is awaited below. + store.put(this.#opToRow(accountAddr, chainIdStr, op)).catch(() => {}) + + if (trimmedId) { + // In-memory trim already identified the op to evict. + store.delete([accountAddr, chainIdStr, trimmedId]).catch(() => {}) + } else { + // The in-memory group is within its cap, but IDB may have accumulated more + // rows than the in-memory limit (e.g. after a startup that only loaded the + // 20-op subset). Count after the put (IDB serializes requests within a tx) + // and evict the oldest row when the group exceeds the hard cap. + const groupRange = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + const count = await store.count(groupRange) + if (count > MAX_IDB_GROUP_SIZE) { + const tsIndex = store.index('by-account-chain-timestamp') + const cursor = await tsIndex.openCursor( + IDBKeyRange.bound( + [accountAddr, chainIdStr, 0], + [accountAddr, chainIdStr, Number.MAX_SAFE_INTEGER] + ) + ) + if (cursor) { + store.delete(cursor.primaryKey as IDBValidKey).catch(() => {}) + } + } + } + + await tx.done + this.#checkQuota() + } + + /** + * Update existing rows in place (status or balance-change updates). + * Uses store.put() per op — no range-delete, only touched rows are written. + */ + async updateOps(ops: SubmittedAccountOp[]): Promise { + if (ops.length === 0) return + + const tx = await this.#openTx('readwrite') + const store = tx.objectStore(this.#storeName) + + try { + for (const op of ops) { + store.put(this.#opToRow(op.accountAddr, op.chainId.toString(), op)).catch(() => {}) + } + } catch (error) { + // Same reasoning as putMultiple: #opToRow throws on an op missing timestamp or + // status, and without an abort the puts already queued would still commit, + // leaving some ops updated and the rest silently skipped. + tx.abort() + tx.done.catch(() => {}) + throw error + } + + await tx.done + } + + /** + * Fetch all ops for a specific (account, chainId) pair (full history, no limit). + * Used for lazy-loading older history during pagination. + * Returns undefined if no ops found (matches existing caller checks). + */ + async getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + const range = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + // Goes through #openTx rather than the db.getAll() shortcut so a dead + // connection is recovered here too. + const tx = await this.#openTx('readonly') + const rows = (await tx.objectStore(this.#storeName).getAll(range)) as IdbAccountOpRow[] + + if (rows.length === 0) return undefined + + rows.sort((a, b) => b.timestamp - a.timestamp) + return rows.map((r) => r.op as SubmittedAccountOp) + } + + /** + * Write ops for a single (account, chainId) pair. + * Deletes existing rows for this pair first, then inserts new ones. + */ + async putOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise { + return this.putMultiple([{ accountAddr, chainId, ops }]) + } + + /** + * Batch write multiple (account, chainId) pairs in a single transaction. + * More efficient than multiple individual puts. + */ + async putMultiple( + records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise { + // Nothing to write — skip opening a transaction at all. Mirrors updateOps. + if (records.length === 0) return + + const tx = await this.#openTx('readwrite') + const store = tx.objectStore(this.#storeName) + + try { + for (const { accountAddr, chainId, ops } of records) { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + this.#writeRecordToStore(store, accountAddr, chainIdStr, this.#dedupeOpsById(ops)) + } + } catch (error) { + // Without this abort, requests already queued in the loop would still + // commit, leaving a partially written store. During migration that is + // unrecoverable: isEmpty() would report false and the ensureMigrated guard + // would skip the retry forever, stranding the rest of the user's history in + // the legacy key. Aborting keeps the batch all-or-nothing. + tx.abort() + // tx.done rejects with the abort; nobody awaits it on this path. + tx.done.catch(() => {}) + throw error + } + + await tx.done + this.#checkQuota() + } + + /** + * Delete all ops for an account across all chains. + */ + async deleteAccount(accountAddr: string): Promise { + const range = IDBKeyRange.bound([accountAddr, '', ''], [accountAddr, RANGE_HIGH, RANGE_HIGH]) + const tx = await this.#openTx('readwrite') + await tx.objectStore(this.#storeName).delete(range) + await tx.done + } + + /** + * Count every row for an account across all chains. + * + * count() on a key range is served from the index structure without reading or + * deserializing any record, so this stays cheap even for a heavy account. + */ + async countOpsForAccount(accountAddr: string): Promise { + const range = IDBKeyRange.bound([accountAddr, '', ''], [accountAddr, RANGE_HIGH, RANGE_HIGH]) + const tx = await this.#openTx('readonly') + + return tx.objectStore(this.#storeName).count(range) + } + + /** + * One-time migration: import all ops from legacy blob storage into IDB. + * After successful import, the caller should remove the key from legacy storage. + */ + async migrateFromStorage(data: InternalAccountsOps): Promise { + const records = Object.entries(data).flatMap(([accountAddr, chainMap]) => + Object.entries(chainMap).map(([chainId, ops]) => ({ accountAddr, chainId, ops })) + ) + return this.putMultiple(records) + } + + /** + * Check if IDB has any data (used to detect if migration is needed). + */ + async isEmpty(): Promise { + const tx = await this.#openTx('readonly') + const count = await tx.objectStore(this.#storeName).count() + return count === 0 + } + + // ────────────────────────────────────────────────────────────────────────────── + // Private helpers + // ────────────────────────────────────────────────────────────────────────────── + + #writeRecordToStore( + store: any, + accountAddr: string, + chainIdStr: string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): void { + // Delete existing rows for this (account, chain), then insert fresh ones + store + .delete( + IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH]) + ) + .catch(() => {}) + for (const op of ops) { + store.put(this.#opToRow(accountAddr, chainIdStr, op)).catch(() => {}) + } + } + + // Drops ops that cannot be stored, then collapses duplicate ids keeping the + // last occurrence. Runs before any write so a bad row never reaches #opToRow, + // whose throw would abandon the batch mid-transaction. + #dedupeOpsById(ops: (SubmittedAccountOp | SubmittedAccountOpLike)[]): StorableOp[] { + const deduped = new Map() + + for (const op of ops) { + if (!isStorableOp(op)) continue + + deduped.set(op.id, op) + } + + return Array.from(deduped.values()) + } + + #checkQuota(): void { + if (!navigator.storage?.estimate) return + + navigator.storage + .estimate() + .then((estimate) => { + if (!estimate.quota || !estimate.usage) return + + const percentUsed = (estimate.usage / estimate.quota) * 100 + const usedMB = (estimate.usage / 1024 / 1024).toFixed(1) + const quotaMB = (estimate.quota / 1024 / 1024).toFixed(1) + + console.log( + `[ActivityIdbStorage] ${this.#storeName} quota: ${usedMB}MB / ${quotaMB}MB (${percentUsed.toFixed(1)}%)` + ) + + if (percentUsed > 80) { + console.warn( + `[ActivityIdbStorage] ${this.#storeName} quota usage high (${percentUsed.toFixed(1)}%)` + ) + } + }) + .catch(() => {}) + } + + #opToRow( + accountAddr: string, + chainIdStr: string, + op: SubmittedAccountOp | SubmittedAccountOpLike + ): IdbAccountOpRow { + if (typeof op.id !== 'string' || !op.id) { + throw new Error('[ActivityIdbStorage] Cannot store op without a valid id') + } + + if (typeof op.timestamp !== 'number') { + throw new Error(`[ActivityIdbStorage] Cannot store op ${op.id} without a valid timestamp`) + } + + if (op.status === undefined) { + throw new Error(`[ActivityIdbStorage] Cannot store op ${op.id} without a valid status`) + } + + return { + accountAddr, + chainId: chainIdStr, + id: op.id, + timestamp: op.timestamp, + status: op.status, + op + } + } +} + +/** + * chrome.storage.local–backed persistence for AccountsOps. + * Used in environments without IndexedDB support (mobile). + * Writes the full in-memory ops blob on every mutation — no row-level granularity. + */ +export class ActivityKeyValueStorage implements IActivityOpsBackend { + // One blob, read whole — there is no window to expand past. + readonly loadsPartially = false + + #storage: IStorageControllerType + #getOps: () => InternalAccountsOps + + /** + * @param storage - The storage controller to read/write from. + * @param getOps - Returns the controller's current in-memory AccountsOps so + * write methods can persist the full up-to-date blob. + */ + constructor(storage: IStorageControllerType, getOps: () => InternalAccountsOps) { + this.#storage = storage + this.#getOps = getOps + } + + // Migration is not needed for storage — data is already in storage. + async ensureMigrated(_g: () => Promise, _r: () => Promise) {} + + async loadStartupOps(): Promise { + return this.#storage.get('accountsOps', {}) + } + + async putSingleOp( + _accountAddr: string, + _chainId: bigint | string, + _op: SubmittedAccountOp, + _trimmedId?: string + ): Promise { + await this.#storage.set('accountsOps', this.#getOps()) + } + + async updateOps(_ops: SubmittedAccountOp[]): Promise { + await this.#storage.set('accountsOps', this.#getOps()) + } + + async getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + const ops = this.#getOps()[accountAddr]?.[chainIdStr] + if (!ops?.length) return undefined + return [...ops].sort((a, b) => b.timestamp - a.timestamp) + } + + async putOpsForAccountAndChain( + _accountAddr: string, + _chainId: bigint | string, + _ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise { + await this.#storage.set('accountsOps', this.#getOps()) + } + + async putMultiple( + _records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise { + await this.#storage.set('accountsOps', this.#getOps()) + } + + async deleteAccount(_accountAddr: string): Promise { + await this.#storage.set('accountsOps', this.#getOps()) + } + + /** + * On this backend the in-memory blob IS the complete history, so summing the group + * lengths is already the true total. + */ + async countOpsForAccount(accountAddr: string): Promise { + const chainMap = this.#getOps()[accountAddr] + if (!chainMap) return 0 + + return Object.values(chainMap).reduce((total, ops) => total + (ops?.length ?? 0), 0) + } +} diff --git a/src/services/storage/idbDatabase.test.ts b/src/services/storage/idbDatabase.test.ts new file mode 100644 index 0000000000..fdfbb8137a --- /dev/null +++ b/src/services/storage/idbDatabase.test.ts @@ -0,0 +1,105 @@ +import 'fake-indexeddb/auto' + +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { openDB } from 'idb' +import { beforeEach, describe, expect, test } from '@jest/globals' + +import { AMBIRE_IDB_SCHEMA } from './idbSchema' +import { openAmbireIdb, resetAmbireIdbForTesting } from './idbDatabase' + +beforeEach(() => { + resetAmbireIdbForTesting() + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange +}) + +describe('openAmbireIdb', () => { + test('opens the database at the version declared in the schema', async () => { + const db = await openAmbireIdb() + expect(db.version).toBe(AMBIRE_IDB_SCHEMA.dbVersion) + }) + + test('creates all stores declared in the schema', async () => { + const db = await openAmbireIdb() + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + expect(db.objectStoreNames.contains(storeDef.storeName)).toBe(true) + } + }) + + test('creates all indexes for each store', async () => { + const db = await openAmbireIdb() + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + const tx = db.transaction(storeDef.storeName, 'readonly') + const store = tx.objectStore(storeDef.storeName) + for (const idx of storeDef.indexes ?? []) { + expect(store.indexNames.contains(idx.name)).toBe(true) + } + await tx.done + } + }) + + test('returns the same promise on repeated calls (singleton)', async () => { + const p1 = openAmbireIdb() + const p2 = openAmbireIdb() + expect(p1).toBe(p2) + await p1 + }) + + test('returns a new promise after resetAmbireIdbForTesting', async () => { + const p1 = openAmbireIdb() + await p1 + resetAmbireIdbForTesting() + global.indexedDB = new IDBFactory() + const p2 = openAmbireIdb() + expect(p1).not.toBe(p2) + await p2 + }) + + test('blocking() closes this connection so another context can upgrade', async () => { + // Regression guard: the handler used to drop the cached promise without ever + // calling db.close(), so an upgrade from a newer app version stayed blocked + // indefinitely and the user had to reload by hand. + const held = await openAmbireIdb() + expect(held.version).toBe(AMBIRE_IDB_SCHEMA.dbVersion) + + // Another context opens the same database at a higher version. This can only + // complete if the connection above yields. + const upgraded = await openDB(AMBIRE_IDB_SCHEMA.dbName, AMBIRE_IDB_SCHEMA.dbVersion + 1) + + expect(upgraded.version).toBe(AMBIRE_IDB_SCHEMA.dbVersion + 1) + upgraded.close() + }) + + test('a fresh install ends up with every store in the manifest', async () => { + // Structure comes from reconcileSchema(), which runs before the versioned + // migration handlers — the handlers themselves only transform existing rows. + const db = await openAmbireIdb() + const storeNames = Array.from(db.objectStoreNames) + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + expect(storeNames).toContain(storeDef.storeName) + } + }) + + test('database is readable and writable after init', async () => { + const db = await openAmbireIdb() + const storeName = AMBIRE_IDB_SCHEMA.stores[0]!.storeName + + const row = { + accountAddr: '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', + chainId: '1', + id: 'test-op', + timestamp: 1000, + status: 0, + op: { id: 'test-op' } + } + + await db.put(storeName, row) + const retrieved = await db.get(storeName, [ + '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', + '1', + 'test-op' + ]) + expect(retrieved).toBeDefined() + expect((retrieved as typeof row).id).toBe('test-op') + }) +}) diff --git a/src/services/storage/idbDatabase.ts b/src/services/storage/idbDatabase.ts new file mode 100644 index 0000000000..60b154aa4e --- /dev/null +++ b/src/services/storage/idbDatabase.ts @@ -0,0 +1,189 @@ +/** + * Global IDB initializer for the 'ambire' database. + * + * Call openAmbireIdb() once at startup and AWAIT it before constructing any controller — + * that await is what guarantees every schema migration finished before a controller reads. + * Mobile never calls it and passes undefined instead, so no platform check belongs here. + * + * Startup order, the schema/data migration distinction, and the rules for writing a + * migration handler are documented in ./README.md. Read it before changing this file. + */ + +import { IDBPDatabase, IDBPTransaction, openDB } from 'idb' + +import { AMBIRE_IDB_SCHEMA, IdbStoreDef } from './idbSchema' + +export type AmbireIdbDatabase = IDBPDatabase + +/** The versionchange transaction handed to migration handlers. */ +export type AmbireIdbUpgradeTransaction = IDBPTransaction + +/** + * Create every store and index in the manifest that does not exist yet. Idempotent, runs on + * every upgrade, and only ever ADDS — see "Structure is declarative" in ./README.md. + */ +export function reconcileSchema( + db: AmbireIdbDatabase, + tx: AmbireIdbUpgradeTransaction, + // Overridable only for tests, so a new store can be exercised without + // mutating the production manifest. + stores: IdbStoreDef[] = AMBIRE_IDB_SCHEMA.stores +): void { + for (const storeDef of stores) { + const store = db.objectStoreNames.contains(storeDef.storeName) + ? tx.objectStore(storeDef.storeName) + : db.createObjectStore(storeDef.storeName, { keyPath: storeDef.keyPath }) + + for (const idx of storeDef.indexes ?? []) { + if (store.indexNames.contains(idx.name)) continue + + store.createIndex(idx.name, idx.keyPath) + console.log(`[AmbireIdb] created index "${idx.name}" on "${storeDef.storeName}"`) + } + } +} + +/** + * Data-migration handlers, keyed by the version they migrate TO. Upgrading v(n) → v(m) runs + * n+1..m in order inside the single onupgradeneeded transaction. Structure is NOT created + * here — reconcileSchema() runs first. + * + * ⚠️ Handlers are SYNCHRONOUS. Chain off the read, never await it: + * store.getAll().then((rows) => rows.forEach((r) => store.put(migrate(r)))) + * Awaiting a non-IDB promise lets the versionchange transaction commit and the writes vanish + * with no error — no unit test catches it, so do not tidy this into an `await`. + * + * The full rules for adding one are in ./README.md under "Writing a migration handler". + */ +export type MigrationHandler = (db: AmbireIdbDatabase, tx: AmbireIdbUpgradeTransaction) => void + +export const migrationHandlers: Record = { + // v0 → v1: initial schema. reconcileSchema() creates accountsOps and its + // indexes; there is no pre-existing data to transform. + 1: () => { + console.log('[AmbireIdb] v1: initial schema applied') + } +} + +/** + * Reconcile structure, then run every data-migration handler in (oldVersion, + * targetVersion] in ascending order. Returns the versions whose handlers ran, + * which lets tests assert the sequence without duplicating the loop. + * + * Exported so that openAmbireIdb() and the tests exercise the same code path. + * + * @param handlers - Overridable only for tests; production always uses the + * module-level registry. + */ +export function applyMigrations( + db: AmbireIdbDatabase, + tx: AmbireIdbUpgradeTransaction, + oldVersion: number, + targetVersion: number, + handlers: Record = migrationHandlers +): number[] { + reconcileSchema(db, tx) + + const applied: number[] = [] + for (let v = oldVersion + 1; v <= targetVersion; v++) { + const handler = handlers[v] + if (!handler) continue + + handler(db, tx) + applied.push(v) + } + + return applied +} + +// ───────────────────────────────────────────────────────────────────────────── +// Singleton +// ───────────────────────────────────────────────────────────────────────────── + +let openPromise: Promise | null = null + +export function openAmbireIdb(): Promise { + if (openPromise) return openPromise + + openPromise = openDB(AMBIRE_IDB_SCHEMA.dbName, AMBIRE_IDB_SCHEMA.dbVersion, { + upgrade(db, oldVersion, newVersion, tx) { + const targetVersion = newVersion ?? AMBIRE_IDB_SCHEMA.dbVersion + console.log( + `[AmbireIdb] Upgrading "${AMBIRE_IDB_SCHEMA.dbName}" v${oldVersion} → v${targetVersion}` + ) + + applyMigrations(db, tx, oldVersion, targetVersion) + }, + + blocked(currentVersion, blockedVersion) { + console.warn( + `[AmbireIdb] Upgrade to v${blockedVersion} is blocked by an open connection at v${currentVersion} (another tab)` + ) + }, + + blocking(currentVersion, blockedVersion) { + // A newer version of the app is trying to open the DB. Close this + // connection so the upgrade can proceed without the user having to + // reload manually. + console.warn( + `[AmbireIdb] This v${currentVersion} connection is blocking an upgrade to v${blockedVersion} — closing` + ) + const prevPromise = openPromise + openPromise = null + prevPromise?.then((db) => db.close()).catch(() => {}) + }, + + terminated() { + // The browser closed the connection on us — storage pressure, the user + // clearing site data, and so on. Nothing above this layer gets told, so + // without dropping the cached promise every later call would keep handing + // out the same dead connection until the whole context restarts. + console.warn('[AmbireIdb] Connection was terminated by the browser — dropping the cache') + openPromise = null + } + }).catch((error) => { + // Allow a subsequent openAmbireIdb() call to retry after a transient failure. + openPromise = null + throw error + }) + + return openPromise +} + +/** + * True when the error means our connection handle is dead while the database + * itself is fine — the connection was closed by blocking(), terminated by the + * browser, or otherwise went away underneath us. + * + * IndexedDB throws InvalidStateError from transaction() as soon as a connection + * has its close-pending flag set, so this is the signal that a caller should + * reopen and retry rather than give up. + */ +export function isClosedConnectionError(error: unknown): boolean { + // Matched by name rather than by instanceof: this arrives as a DOMException in + // browsers and as a library-defined error class under fake-indexeddb, and + // neither is reliably an instance of Error. + if (typeof error !== 'object' || error === null) return false + + return (error as { name?: unknown }).name === 'InvalidStateError' +} + +/** + * Drop the cached connection so the next openAmbireIdb() opens a fresh one. + * + * blocking() and terminated() already do this when they fire, but a connection can + * also die without either event. In that case the cache still resolves to the dead + * handle, so a caller recovering from isClosedConnectionError must invalidate first + * — otherwise it reconnects to the same dead connection and fails again. + */ +export function invalidateAmbireIdbConnection(): void { + openPromise = null +} + +/** + * Reset the singleton — for use in tests only. + * Call before replacing global.indexedDB with a fresh IDBFactory. + */ +export function resetAmbireIdbForTesting(): void { + openPromise = null +} diff --git a/src/services/storage/idbIntegration.test.ts b/src/services/storage/idbIntegration.test.ts new file mode 100644 index 0000000000..792db5c1fe --- /dev/null +++ b/src/services/storage/idbIntegration.test.ts @@ -0,0 +1,843 @@ +/** + * IDB integration tests using a self-contained dummy controller. + * + * Purpose: verify the infrastructure pattern — not phishing-specific logic. + * - Dynamic path: IDB is available (extension / web) + * - Static path: IDB is not available (mobile / key-value fallback) + * - Schema upgrade: v1 → v2 runs the migration handler without data loss + * + * The DummyController defined below is the canonical template for wiring a + * new controller to IDB. It mirrors the three steps every controller's #load() + * must follow: + * 1. await backend.ensureMigrated(...) ← migration before any read + * 2. this.state = await backend.loadSnapshot() + * 3. mutations call backend.saveSnapshot(newState) + */ + +import 'fake-indexeddb/auto' + +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { openDB } from 'idb' +import { beforeEach, describe, expect, jest, test } from '@jest/globals' + +import { + AmbireIdbDatabase, + AmbireIdbUpgradeTransaction, + applyMigrations, + MigrationHandler, + migrationHandlers, + openAmbireIdb, + reconcileSchema, + resetAmbireIdbForTesting +} from './idbDatabase' +import { AMBIRE_IDB_SCHEMA } from './idbSchema' + +// ───────────────────────────────────────────────────────────────────────────── +// Dummy backend interface + implementations +// +// These mirror the IActivityOpsBackend / IPhishingOpsBackend pattern but with +// a minimal generic payload so the tests stay focused on the lifecycle, not +// domain logic. +// ───────────────────────────────────────────────────────────────────────────── + +interface DummyState { + version: number + data: string +} + +const DEFAULT_DUMMY_STATE: DummyState = { version: 0, data: '' } +// Reuses the 'phishing' store name, but inside the isolated test DB created below — +// these tests never touch the production database. +const DUMMY_STORE = 'phishing' +const DUMMY_KEY = 'dummy-snapshot' + +interface IDummyBackend { + isEmpty(): Promise + migrateFromStorage(state: DummyState): Promise + ensureMigrated( + getLegacy: () => Promise, + removeLegacy: () => Promise + ): Promise + load(): Promise + save(state: DummyState): Promise +} + +/** IDB-backed backend — used in web/extension environments. */ +class DummyIdbBackend implements IDummyBackend { + #db: AmbireIdbDatabase + + constructor(db: AmbireIdbDatabase) { + this.#db = db + } + + async isEmpty(): Promise { + return (await this.#db.count(DUMMY_STORE)) === 0 + } + + async migrateFromStorage(state: DummyState): Promise { + await this.save(state) + } + + async ensureMigrated( + getLegacy: () => Promise, + removeLegacy: () => Promise + ): Promise { + const empty = await this.isEmpty() + if (!empty) return + const legacy = await getLegacy() + if (!legacy.version && !legacy.data) return + await this.migrateFromStorage(legacy) + await removeLegacy() + } + + async load(): Promise { + const row = (await this.#db.get(DUMMY_STORE, DUMMY_KEY)) as + | (DummyState & { id: string }) + | undefined + if (!row) return { ...DEFAULT_DUMMY_STATE } + const { id: _id, ...state } = row + return state + } + + async save(state: DummyState): Promise { + await this.#db.put(DUMMY_STORE, { id: DUMMY_KEY, ...state }) + } +} + +/** Key-value–backed backend — used on mobile (no IDB). */ +class DummyKeyValueBackend implements IDummyBackend { + #store: Record + + constructor(store: Record) { + this.#store = store + } + + // Data already lives in its final location, so nothing here migrates. + async isEmpty(): Promise { + return false + } + + async migrateFromStorage(_state: DummyState): Promise {} + + async ensureMigrated(_getLegacy: () => Promise, _remove: () => Promise) {} + + async load(): Promise { + return { ...(this.#store['dummy'] ?? DEFAULT_DUMMY_STATE) } + } + + async save(state: DummyState): Promise { + this.#store['dummy'] = state + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// DummyController +// +// Template for any controller that follows the IDB backend pattern. +// Replace DummyState / IDummyBackend with the real types when wiring a new +// controller. The three steps in #load() must remain in this exact order. +// ───────────────────────────────────────────────────────────────────────────── + +class DummyController { + #backend: IDummyBackend + state: DummyState = { ...DEFAULT_DUMMY_STATE } + + constructor(backend: IDummyBackend) { + this.#backend = backend + } + + /** + * Mirrors a real controller's #load(): + * step 1 — migrate legacy data before reading anything + * step 2 — load startup state from the backend + */ + async load( + getLegacy: () => Promise, + removeLegacy: () => Promise + ): Promise { + await this.#backend.ensureMigrated(getLegacy, removeLegacy) + this.state = await this.#backend.load() + } + + async update(newState: DummyState): Promise { + this.state = newState + await this.#backend.save(newState) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Setup +// ───────────────────────────────────────────────────────────────────────────── + +let db: AmbireIdbDatabase + +// Opens an isolated test DB rather than going through openAmbireIdb(), so the +// DummyController scenarios stay independent of the production manifest and do not +// need updating every time a store or version is added to it. +async function openTestDb(): Promise { + return openDB('integration-test', 1, { + upgrade(d) { + d.createObjectStore('accountsOps', { keyPath: ['accountAddr', 'chainId', 'id'] }) + d.createObjectStore('phishing', { keyPath: 'id' }) + } + }) +} + +beforeEach(async () => { + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + ;(global as any).navigator = {} + db = await openTestDb() +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Dynamic controller (IDB path) +// ───────────────────────────────────────────────────────────────────────────── + +describe('Dynamic controller — IDB path', () => { + test('state reflects legacy storage after first init', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + const legacy: DummyState = { version: 3, data: 'migrated-value' } + + await ctrl.load( + async () => legacy, + async () => {} + ) + + expect(ctrl.state.version).toBe(3) + expect(ctrl.state.data).toBe('migrated-value') + }) + + test('migration runs before load — ordering guarantee', async () => { + const backend = new DummyIdbBackend(db) + const ctrl = new DummyController(backend) + const legacy: DummyState = { version: 5, data: 'must-arrive-before-load' } + + // #load() awaits ensureMigrated, then reads — data must be present + await ctrl.load( + async () => legacy, + async () => {} + ) + + expect(ctrl.state.data).toBe('must-arrive-before-load') + }) + + test('legacy storage key is removed after migration', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + const removeSpy = jest.fn(async () => {}) + + await ctrl.load(async () => ({ version: 1, data: 'x' }), removeSpy) + + expect(removeSpy).toHaveBeenCalledTimes(1) + }) + + test('second init (service worker restart) skips migration and reads IDB', async () => { + const ctrl1 = new DummyController(new DummyIdbBackend(db)) + await ctrl1.load( + async () => ({ version: 7, data: 'original' }), + async () => {} + ) + + // Simulate restart: new controller instance, same db + const getLegacySpy = jest.fn(async () => ({ version: 99, data: 'stale' })) + const ctrl2 = new DummyController(new DummyIdbBackend(db)) + await ctrl2.load(getLegacySpy, async () => {}) + + expect(getLegacySpy).not.toHaveBeenCalled() + expect(ctrl2.state.version).toBe(7) + }) + + test('fresh install with no legacy data loads default state', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + expect(ctrl.state).toEqual(DEFAULT_DUMMY_STATE) + }) + + test('update persists state so the next init reads the saved value', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + await ctrl.update({ version: 2, data: 'saved' }) + + const ctrl2 = new DummyController(new DummyIdbBackend(db)) + await ctrl2.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + expect(ctrl2.state.data).toBe('saved') + }) + + test('propagates error when getLegacy throws during migration', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + const boom = new Error('storage read failed') + + await expect( + ctrl.load( + async () => { + throw boom + }, + async () => {} + ) + ).rejects.toThrow('storage read failed') + }) + + test('IDB data is intact when removeLegacy throws after save — next load skips migration', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + const legacy: DummyState = { version: 4, data: 'migrated' } + + // removeLegacy fails, but the save to IDB already completed + await expect( + ctrl.load( + async () => legacy, + async () => { + throw new Error('cleanup failed') + } + ) + ).rejects.toThrow('cleanup failed') + + // IDB is now non-empty — the next load must find the data and skip migration + const getLegacySpy = jest.fn(async (): Promise => ({ ...DEFAULT_DUMMY_STATE })) + const ctrl2 = new DummyController(new DummyIdbBackend(db)) + await ctrl2.load(getLegacySpy, async () => {}) + + expect(getLegacySpy).not.toHaveBeenCalled() + expect(ctrl2.state.data).toBe('migrated') + }) + + test('skips migration when version is 0 and data is empty — guard treats both as falsy', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + const removeSpy = jest.fn(async () => {}) + + await ctrl.load(async () => ({ version: 0, data: '' }), removeSpy) + + expect(removeSpy).not.toHaveBeenCalled() + expect(ctrl.state).toEqual(DEFAULT_DUMMY_STATE) + }) + + test('migrates when version is 0 but data is non-empty — only the falsy-both case is skipped', async () => { + // The guard is `!version && !data` — non-empty data triggers migration even at version 0. + const ctrl = new DummyController(new DummyIdbBackend(db)) + + await ctrl.load( + async () => ({ version: 0, data: 'payload-only' }), + async () => {} + ) + + expect(ctrl.state.data).toBe('payload-only') + }) + + test('concurrent load() calls complete without data loss', async () => { + const legacy: DummyState = { version: 1, data: 'concurrent' } + const ctrl1 = new DummyController(new DummyIdbBackend(db)) + const ctrl2 = new DummyController(new DummyIdbBackend(db)) + + // Both instances start before either has written — both will see isEmpty()=true + // and run migration. The second put overwrites with identical data. + await Promise.all([ + ctrl1.load( + async () => legacy, + async () => {} + ), + ctrl2.load( + async () => legacy, + async () => {} + ) + ]) + + expect(ctrl1.state.data).toBe('concurrent') + expect(ctrl2.state.data).toBe('concurrent') + + // IDB holds exactly one record (the second put was idempotent) + const backend = new DummyIdbBackend(db) + expect(await backend.load()).toEqual(legacy) + }) + + test('three sequential update() calls — only the last value survives reload', async () => { + const ctrl = new DummyController(new DummyIdbBackend(db)) + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + await ctrl.update({ version: 1, data: 'first' }) + await ctrl.update({ version: 2, data: 'second' }) + await ctrl.update({ version: 3, data: 'third' }) + + const ctrl2 = new DummyController(new DummyIdbBackend(db)) + await ctrl2.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + expect(ctrl2.state.version).toBe(3) + expect(ctrl2.state.data).toBe('third') + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Static controller (key-value storage path) +// ───────────────────────────────────────────────────────────────────────────── + +describe('Static controller — key-value storage path', () => { + test('loads existing state from storage without migration', async () => { + const store = { dummy: { version: 4, data: 'from-storage' } } + const ctrl = new DummyController(new DummyKeyValueBackend(store)) + + const getLegacySpy = jest.fn(async () => ({ ...DEFAULT_DUMMY_STATE })) + const removeSpy = jest.fn(async () => {}) + await ctrl.load(getLegacySpy, removeSpy) + + expect(ctrl.state.version).toBe(4) + expect(ctrl.state.data).toBe('from-storage') + // ensureMigrated is a no-op — neither callback is touched + expect(getLegacySpy).not.toHaveBeenCalled() + expect(removeSpy).not.toHaveBeenCalled() + }) + + test('loads default state on a brand-new install', async () => { + const store = {} + const ctrl = new DummyController(new DummyKeyValueBackend(store)) + + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + expect(ctrl.state).toEqual(DEFAULT_DUMMY_STATE) + }) + + test('update persists state to storage', async () => { + const store: Record = {} + const ctrl = new DummyController(new DummyKeyValueBackend(store)) + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + await ctrl.update({ version: 1, data: 'written' }) + + expect(store['dummy']).toEqual({ version: 1, data: 'written' }) + }) + + test('three sequential update() calls — only the last value survives in storage', async () => { + const store: Record = {} + const ctrl = new DummyController(new DummyKeyValueBackend(store)) + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + await ctrl.update({ version: 1, data: 'first' }) + await ctrl.update({ version: 2, data: 'second' }) + await ctrl.update({ version: 3, data: 'third' }) + + expect(store['dummy']).toEqual({ version: 3, data: 'third' }) + }) + + test('mutating the loaded state does not affect subsequent loads', async () => { + const store = { dummy: { version: 1, data: 'original' } } + const ctrl = new DummyController(new DummyKeyValueBackend(store)) + await ctrl.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + // Mutate the controller's in-memory state after load + ctrl.state.data = 'mutated' + + // A new controller reading the same store must see the original value + const ctrl2 = new DummyController(new DummyKeyValueBackend(store)) + await ctrl2.load( + async () => ({ ...DEFAULT_DUMMY_STATE }), + async () => {} + ) + + expect(ctrl2.state.data).toBe('original') + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Schema manifest ↔ migration handler consistency +// +// These are static guards. They fail at CI time the moment dbVersion, the +// handler registry, and the migrations manifest drift apart — which would +// otherwise only surface at runtime as a missing store on a user's machine. +// ───────────────────────────────────────────────────────────────────────────── + +describe('Schema manifest ↔ migration handler consistency', () => { + test('every version from 1..dbVersion has a migration handler', () => { + const missing: number[] = [] + for (let v = 1; v <= AMBIRE_IDB_SCHEMA.dbVersion; v++) { + if (!migrationHandlers[v]) missing.push(v) + } + expect(missing).toEqual([]) + }) + + test('no migration handler is registered above dbVersion', () => { + const registered = Object.keys(migrationHandlers).map(Number) + const stray = registered.filter((v) => v > AMBIRE_IDB_SCHEMA.dbVersion) + expect(stray).toEqual([]) + }) + + test('migrations manifest describes a contiguous 0 → dbVersion chain', () => { + const sorted = [...AMBIRE_IDB_SCHEMA.migrations].sort((a, b) => a.toVersion - b.toVersion) + + expect(sorted[0]?.fromVersion).toBe(0) + expect(sorted[sorted.length - 1]?.toVersion).toBe(AMBIRE_IDB_SCHEMA.dbVersion) + + sorted.forEach((migration, i) => { + // Each entry advances exactly one version, and picks up where the last left off + expect(migration.toVersion).toBe(migration.fromVersion + 1) + if (i > 0) expect(migration.fromVersion).toBe(sorted[i - 1]?.toVersion) + }) + }) + + test('store names in the manifest are unique', () => { + const names = AMBIRE_IDB_SCHEMA.stores.map((s) => s.storeName) + expect(names).toHaveLength(new Set(names).size) + }) + + test('index names within each store are unique', () => { + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + const names = (storeDef.indexes ?? []).map((i) => i.name) + expect(names).toHaveLength(new Set(names).size) + } + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Schema upgrade +// +// These exercise the real reconcileSchema() and applyMigrations() exported from +// idbDatabase.ts rather than a copy of the loop, so a regression in the +// production migration path fails here. +// ───────────────────────────────────────────────────────────────────────────── + +describe('Schema upgrade', () => { + test('reconcileSchema adds a store introduced by a later manifest version', async () => { + // 'notifications' is deliberately NOT in the manifest — it stands in for a + // future addition. Using a real manifest store here would prove nothing, since + // reconcileSchema would have created it on the first pass anyway. + const FUTURE_STORE = 'notifications' + + const currentDb = await openDB('ambire-dummy-upgrade', 1, { + upgrade(d, _oldVersion, _newVersion, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction) + } + }) + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + expect(currentDb.objectStoreNames.contains(storeDef.storeName)).toBe(true) + } + expect(currentDb.objectStoreNames.contains(FUTURE_STORE)).toBe(false) + currentDb.close() + + // Now the manifest gains a store + const nextStores = [...AMBIRE_IDB_SCHEMA.stores, { storeName: FUTURE_STORE, keyPath: 'id' }] + const upgraded = await openDB('ambire-dummy-upgrade', 2, { + upgrade(d, _oldVersion, _newVersion, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction, nextStores) + } + }) + + // Pre-existing stores survive and the new one appears + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + expect(upgraded.objectStoreNames.contains(storeDef.storeName)).toBe(true) + } + expect(upgraded.objectStoreNames.contains(FUTURE_STORE)).toBe(true) + upgraded.close() + }) + + test('reconcileSchema adds an index to a store that already exists', async () => { + // The trap this closes: a store created before an index was declared. A + // create-store-only handler skips existing stores, so upgrading users would + // never receive the new index while fresh installs would. + const bare = await openDB('ambire-dummy-index-add', 1, { + upgrade(d) { + d.createObjectStore('accountsOps', { keyPath: ['accountAddr', 'chainId', 'id'] }) + } + }) + expect([...bare.transaction('accountsOps').store.indexNames]).toEqual([]) + bare.close() + + const upgraded = await openDB('ambire-dummy-index-add', 2, { + upgrade(d, _oldVersion, _newVersion, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction) + } + }) + + const indexNames = [...upgraded.transaction('accountsOps').store.indexNames] + const declared = ( + AMBIRE_IDB_SCHEMA.stores.find((s) => s.storeName === 'accountsOps')?.indexes ?? [] + ).map((i) => i.name) + expect(declared.length).toBeGreaterThan(0) + for (const name of declared) expect(indexNames).toContain(name) + upgraded.close() + }) + + test('reconcileSchema is idempotent — a second run changes nothing', async () => { + const first = await openDB('ambire-dummy-idempotent', 1, { + upgrade(d, _o, _n, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction) + } + }) + const before = [...first.transaction('accountsOps').store.indexNames].sort() + first.close() + + const second = await openDB('ambire-dummy-idempotent', 2, { + upgrade(d, _o, _n, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction) + } + }) + const after = [...second.transaction('accountsOps').store.indexNames].sort() + + expect(after).toEqual(before) + expect(second.objectStoreNames.contains('accountsOps')).toBe(true) + second.close() + }) + + test('v1 data survives the upgrade and a newly added store is empty and writable', async () => { + const ACCOUNT = '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + + const v1Db = await openDB('ambire-dummy-data-upgrade', 1, { + upgrade(d, _o, _n, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction) + } + }) + await v1Db.put('accountsOps', { + accountAddr: ACCOUNT, + chainId: '1', + id: 'op-1', + timestamp: 1000, + status: 'Success', + op: {} + }) + v1Db.close() + + const nextStores = [...AMBIRE_IDB_SCHEMA.stores, { storeName: 'phishing', keyPath: 'id' }] + const v2Db = await openDB('ambire-dummy-data-upgrade', 2, { + upgrade(d, _o, _n, tx) { + reconcileSchema(d, tx as AmbireIdbUpgradeTransaction, nextStores) + } + }) + + // Existing v1 row must be intact + const row = await v2Db.get('accountsOps', [ACCOUNT, '1', 'op-1']) + expect(row).toBeDefined() + expect((row as any).id).toBe('op-1') + + // New store must be empty but functional + expect(await v2Db.count('phishing')).toBe(0) + const ctrl = new DummyController(new DummyIdbBackend(v2Db)) + await ctrl.load( + async () => ({ version: 1, data: 'post-upgrade' }), + async () => {} + ) + expect(ctrl.state.data).toBe('post-upgrade') + + v2Db.close() + }) + + // openAmbireIdb()'s own store/index/singleton guarantees are covered in + // idbDatabase.test.ts — not repeated here. +}) + +// ───────────────────────────────────────────────────────────────────────────── +// applyMigrations — the production loop +// +// Handlers are injected so multi-step sequences can be exercised while the +// manifest stays at its real version. The loop itself is the production one. +// ───────────────────────────────────────────────────────────────────────────── + +describe('applyMigrations', () => { + /** Runs applyMigrations inside a real versionchange transaction. */ + async function upgradeWith( + dbName: string, + version: number, + handlers: Record + ): Promise<{ db: AmbireIdbDatabase; applied: number[] }> { + let applied: number[] = [] + const db = await openDB(dbName, version, { + upgrade(d, oldVersion, newVersion, tx) { + applied = applyMigrations( + d, + tx as AmbireIdbUpgradeTransaction, + oldVersion, + newVersion ?? version, + handlers + ) + } + }) + return { db, applied } + } + + test('runs handlers for (oldVersion, targetVersion] in ascending order', async () => { + const order: number[] = [] + const handlers: Record = { + 1: () => order.push(1), + 2: () => order.push(2), + 3: () => order.push(3) + } + + const { db, applied } = await upgradeWith('ambire-apply-order', 3, handlers) + + expect(order).toEqual([1, 2, 3]) + expect(applied).toEqual([1, 2, 3]) + db.close() + }) + + test('skips the handler for oldVersion itself on a partial upgrade', async () => { + const order: number[] = [] + const handlers: Record = { + 1: () => order.push(1), + 2: () => order.push(2), + 3: () => order.push(3) + } + + const first = await upgradeWith('ambire-apply-partial', 1, handlers) + expect(first.applied).toEqual([1]) + first.db.close() + + // v1 → v3 must run 2 and 3 only — never re-run 1 + const second = await upgradeWith('ambire-apply-partial', 3, handlers) + expect(second.applied).toEqual([2, 3]) + expect(order).toEqual([1, 2, 3]) + second.db.close() + }) + + test('gaps in the handler registry are skipped without throwing', async () => { + // A purely additive schema change needs no data handler, so a gap is normal. + const handlers: Record = { 1: () => {} } + + const { db, applied } = await upgradeWith('ambire-apply-gap', 3, handlers) + + expect(applied).toEqual([1]) + // Structure still complete — reconcileSchema does not depend on handlers + expect(db.objectStoreNames.contains('accountsOps')).toBe(true) + db.close() + }) + + test('reconcileSchema runs before handlers so a handler can write to a new store', async () => { + // Ordering guarantee: a data-migration handler must be able to read and + // write stores introduced by the same upgrade. + const handlers: Record = { + 1: (_db, tx) => { + tx.objectStore('accountsOps').put({ + accountAddr: '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', + chainId: '1', + id: 'written-by-handler', + timestamp: 1, + status: 'success', + op: {} + }) + } + } + + const { db } = await upgradeWith('ambire-apply-ordering', 1, handlers) + + const row = await db.get('accountsOps', [ + '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', + '1', + 'written-by-handler' + ]) + expect(row).toBeDefined() + db.close() + }) + + test('a handler can read and transform rows written by an earlier version', async () => { + // This is the entire reason handlers exist, so it needs to be provably possible. + // Handlers are synchronous, so the read cannot be awaited — it has to be chained. + // The versionchange transaction stays alive across microtasks, so a .then() that + // issues further requests still lands inside the same upgrade. + const ACCOUNT = '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + + const first = await upgradeWith('ambire-apply-transform', 1, { 1: () => {} }) + await first.db.put('accountsOps', { + accountAddr: ACCOUNT, + chainId: '1', + id: 'op-1', + timestamp: 1000, + status: 'stale-status', + op: {} + }) + first.db.close() + + const handlers: Record = { + 1: () => {}, + 2: (_db, tx) => { + const store = tx.objectStore('accountsOps') + store.getAll().then((rows: any[]) => { + rows.forEach((row) => store.put({ ...row, status: 'migrated-status' })) + }) + } + } + + const second = await upgradeWith('ambire-apply-transform', 2, handlers) + + const row = await second.db.get('accountsOps', [ACCOUNT, '1', 'op-1']) + expect((row as any).status).toBe('migrated-status') + second.db.close() + }) + + test('runs no handlers when oldVersion already equals the target', async () => { + const handlers: Record = { 1: () => {}, 2: () => {} } + + const first = await upgradeWith('ambire-apply-noop', 2, handlers) + expect(first.applied).toEqual([1, 2]) + first.db.close() + + // Re-opening at the same version does not trigger upgrade() at all + const reopened = await openDB('ambire-apply-noop', 2) + expect(reopened.version).toBe(2) + reopened.close() + }) +}) + +// The happy-path singleton behaviour (same promise on repeated calls, new promise +// after a reset) is covered in idbDatabase.test.ts. What is left here is the +// failure path, which that suite does not exercise. +describe('openAmbireIdb singleton', () => { + test('resets openPromise on failure so a subsequent call can retry', async () => { + // Pre-open the production DB at a version HIGHER than the schema so that + // openAmbireIdb() requesting the lower version receives a VersionError. + // The .catch() handler must reset openPromise = null so the retry succeeds. + resetAmbireIdbForTesting() + try { + const higherVersion = AMBIRE_IDB_SCHEMA.dbVersion + 1 + const prelim = await openDB(AMBIRE_IDB_SCHEMA.dbName, higherVersion, { + upgrade(d) { + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + if (!d.objectStoreNames.contains(storeDef.storeName)) { + d.createObjectStore(storeDef.storeName, { keyPath: storeDef.keyPath }) + } + } + } + }) + prelim.close() + + // openAmbireIdb() at the lower schema version should fail with VersionError + await expect(openAmbireIdb()).rejects.toThrow() + + // openPromise was reset to null by the .catch() — reset the IDB factory so + // the retry opens a fresh database at the production schema version + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + + const retried = await openAmbireIdb() + expect(retried).toBeDefined() + for (const storeDef of AMBIRE_IDB_SCHEMA.stores) { + expect(retried.objectStoreNames.contains(storeDef.storeName)).toBe(true) + } + retried.close() + } finally { + resetAmbireIdbForTesting() + } + }) +}) diff --git a/src/services/storage/idbSchema.ts b/src/services/storage/idbSchema.ts new file mode 100644 index 0000000000..fe81aec83b --- /dev/null +++ b/src/services/storage/idbSchema.ts @@ -0,0 +1,85 @@ +/** + * Static IDB schema manifest — the single source of truth for the STRUCTURE of the + * 'ambire' database. + * + * All stores, keyPaths, and indexes are declared here. reconcileSchema() in + * idbDatabase.ts creates anything in this manifest that does not exist yet, on + * every upgrade and idempotently. Structure is therefore declarative: a purely + * additive change needs no hand-written migration code. + * + * Rules for making schema changes: + * 1. Add or modify a store definition below. + * 2. Bump dbVersion by 1 — reconcileSchema only runs during an upgrade, so + * without a version bump existing installs never pick the change up. + * 3. Add an entry to `migrations` below describing what changed. + * 4. Add a handler to `migrationHandlers` in idbDatabase.ts keyed by the new + * version. It may be a no-op: handlers exist for transforming EXISTING ROWS, + * not for creating stores or indexes. An entry is still required so that a + * version bump is always deliberate — a test enforces this. + * + * Never remove a migration entry or handler — the chain must stay intact so users + * upgrading from any prior version reach the current schema. + * + * Note: `migrations` is documentation only. Nothing reads it at runtime; a test + * checks that it forms a contiguous 0 → dbVersion chain so it cannot silently + * drift out of step with the real version. + */ + +export interface IdbIndexDef { + name: string + keyPath: string | string[] +} + +export interface IdbStoreDef { + storeName: string + keyPath: string | string[] + indexes?: IdbIndexDef[] +} + +export interface IdbMigration { + fromVersion: number + toVersion: number + description: string +} + +export interface IdbSchema { + dbName: string + dbVersion: number + stores: IdbStoreDef[] + migrations: IdbMigration[] +} + +// ───────────────────────────────────────────────────────────────────────────── +// Schema +// ───────────────────────────────────────────────────────────────────────────── + +export const AMBIRE_IDB_SCHEMA: IdbSchema = { + dbName: 'ambire', + dbVersion: 1, + stores: [ + { + storeName: 'accountsOps', + keyPath: ['accountAddr', 'chainId', 'id'], + indexes: [ + { + name: 'by-account-chain-timestamp', + keyPath: ['accountAddr', 'chainId', 'timestamp'] + }, + { + name: 'by-account-chain-status', + keyPath: ['accountAddr', 'chainId', 'status'] + } + ] + } + ], + // Human-readable changelog of the schema. Not read at runtime — the executable + // counterparts are `stores` above (structure) and `migrationHandlers` in + // idbDatabase.ts keyed by toVersion (row transformations). + migrations: [ + { + fromVersion: 0, + toVersion: 1, + description: 'Initial schema: accountsOps store with timestamp and status indexes' + } + ] +} diff --git a/src/services/storage/phishingIdb.test.ts b/src/services/storage/phishingIdb.test.ts new file mode 100644 index 0000000000..e0630d2845 --- /dev/null +++ b/src/services/storage/phishingIdb.test.ts @@ -0,0 +1,407 @@ +import 'fake-indexeddb/auto' + +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { openDB } from 'idb' +import { beforeEach, describe, expect, jest, test } from '@jest/globals' + +import { + DEFAULT_PHISHING_SNAPSHOT, + PhishingIdbStorage, + PhishingKeyValueStorage, + PhishingSnapshot +} from './phishingIdb' +import { AmbireIdbDatabase } from './idbDatabase' + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +function makeSnapshot(overrides: Partial = {}): PhishingSnapshot { + return { + version: 1, + updatedAt: 1000, + domains: ['phishing.example.com', 'scam.io'], + addresses: ['0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef'], + ...overrides + } +} + +/** + * Minimal in-memory IStorageController stub — only the methods used by + * PhishingKeyValueStorage are needed here. + */ +function makeStorageMock(initial: Record = {}) { + const store: Record = { ...initial } + return { + get: async (key: string, defaultValue: any) => + key in store ? JSON.parse(JSON.stringify(store[key])) : defaultValue, + set: async (key: string, value: any) => { + store[key] = JSON.parse(JSON.stringify(value)) + return null + }, + remove: async (key: string) => { + delete store[key] + return null + }, + _store: store + } +} + +let db: AmbireIdbDatabase + +// Opens a minimal isolated DB with only the phishing store. Deliberately does NOT +// use openAmbireIdb(): the phishing store is not in AMBIRE_IDB_SCHEMA, because +// adding it would mean an unrollbackable dbVersion bump for a store nothing reads +// yet. Switch these to openAmbireIdb() in the change that wires PhishingController. +async function openTestDb(): Promise { + return openDB('phishing-unit-test', 1, { + upgrade(d) { + d.createObjectStore('phishing', { keyPath: 'id' }) + } + }) +} + +beforeEach(async () => { + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + // checkQuota() reads navigator.storage — stub it to avoid ReferenceError in Node. + ;(global as any).navigator = {} + db = await openTestDb() +}) + +// ───────────────────────────────────────────────────────────────────────────── +// PhishingIdbStorage (IDB / "dynamic" backend) +// ───────────────────────────────────────────────────────────────────────────── + +describe('PhishingIdbStorage', () => { + describe('isEmpty', () => { + test('returns true on a fresh store', async () => { + const store = new PhishingIdbStorage(db) + expect(await store.isEmpty()).toBe(true) + }) + + test('returns false after a snapshot is saved', async () => { + const store = new PhishingIdbStorage(db) + await store.saveSnapshot(makeSnapshot()) + expect(await store.isEmpty()).toBe(false) + }) + }) + + describe('loadSnapshot', () => { + test('returns DEFAULT_PHISHING_SNAPSHOT on an empty store', async () => { + const store = new PhishingIdbStorage(db) + expect(await store.loadSnapshot()).toEqual(DEFAULT_PHISHING_SNAPSHOT) + }) + + test('returns the saved snapshot after saveSnapshot', async () => { + const store = new PhishingIdbStorage(db) + const snap = makeSnapshot() + await store.saveSnapshot(snap) + expect(await store.loadSnapshot()).toEqual(snap) + }) + + test('does not expose the internal id field in the returned snapshot', async () => { + const store = new PhishingIdbStorage(db) + await store.saveSnapshot(makeSnapshot()) + const result = await store.loadSnapshot() + expect(result).not.toHaveProperty('id') + }) + + test('returns independent objects — mutations to the result do not affect IDB', async () => { + const store = new PhishingIdbStorage(db) + await store.saveSnapshot(makeSnapshot()) + const result = await store.loadSnapshot() + result.domains.push('injected.evil') + const reloaded = await store.loadSnapshot() + expect(reloaded.domains).not.toContain('injected.evil') + }) + }) + + describe('saveSnapshot', () => { + test('round-trips every field', async () => { + const store = new PhishingIdbStorage(db) + const snap = makeSnapshot({ + version: 42, + updatedAt: 9999, + domains: ['evil.com', 'phish.io'], + addresses: ['0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1'] + }) + + await store.saveSnapshot(snap) + + expect(await store.loadSnapshot()).toEqual(snap) + }) + + test('overwrites previous data on a second save', async () => { + const store = new PhishingIdbStorage(db) + await store.saveSnapshot(makeSnapshot({ version: 1, domains: ['old.com'] })) + await store.saveSnapshot(makeSnapshot({ version: 2, domains: ['new.com'] })) + const result = await store.loadSnapshot() + expect(result.version).toBe(2) + expect(result.domains).toEqual(['new.com']) + }) + + test('persists empty domains and addresses arrays without error', async () => { + const store = new PhishingIdbStorage(db) + await store.saveSnapshot(makeSnapshot({ domains: [], addresses: [] })) + const result = await store.loadSnapshot() + expect(result.domains).toEqual([]) + expect(result.addresses).toEqual([]) + }) + }) + + describe('migrateFromStorage', () => { + // migrateFromStorage delegates to saveSnapshot, which the block above covers + // in full. One round-trip is enough to pin the delegation. + test('imports a legacy snapshot and leaves the store non-empty', async () => { + const store = new PhishingIdbStorage(db) + const legacy = makeSnapshot({ + version: 7, + updatedAt: 5000, + domains: ['legacy-phish.com'], + addresses: ['0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'] + }) + + await store.migrateFromStorage(legacy) + + expect(await store.loadSnapshot()).toEqual(legacy) + expect(await store.isEmpty()).toBe(false) + }) + }) + + describe('ensureMigrated', () => { + test('migrates when IDB is empty and storage has meaningful data', async () => { + const store = new PhishingIdbStorage(db) + const legacy = makeSnapshot({ version: 3 }) + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated(async () => legacy, removeSpy) + + const result = await store.loadSnapshot() + expect(result.version).toBe(3) + expect(result.domains).toEqual(legacy.domains) + expect(removeSpy).toHaveBeenCalledTimes(1) + }) + + test('does not migrate when IDB already has a snapshot', async () => { + const store = new PhishingIdbStorage(db) + const existing = makeSnapshot({ version: 10, domains: ['already-there.com'] }) + await store.saveSnapshot(existing) + + const getStoredSpy = jest.fn(async () => makeSnapshot({ version: 99 })) + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated(getStoredSpy, removeSpy) + + // Neither callback should be called — IDB already has data + expect(getStoredSpy).not.toHaveBeenCalled() + expect(removeSpy).not.toHaveBeenCalled() + + // IDB data is unchanged + const result = await store.loadSnapshot() + expect(result.version).toBe(10) + }) + + test('does not migrate when storage also has no meaningful data', async () => { + const store = new PhishingIdbStorage(db) + const removeSpy = jest.fn(async () => {}) + + // Storage returns the default (version=0, empty arrays) + await store.ensureMigrated(async () => ({ ...DEFAULT_PHISHING_SNAPSHOT }), removeSpy) + + expect(await store.isEmpty()).toBe(true) + expect(removeSpy).not.toHaveBeenCalled() + }) + + test('migrates when version is 0 but domains is non-empty — only the all-falsy case is skipped', async () => { + // Guard: `!stored.version && !stored.domains.length && !stored.addresses.length` + // Non-empty domains trigger migration even at version 0. + const store = new PhishingIdbStorage(db) + + await store.ensureMigrated( + async () => makeSnapshot({ version: 0, domains: ['early-phish.com'], addresses: [] }), + async () => {} + ) + + const result = await store.loadSnapshot() + expect(result.domains).toEqual(['early-phish.com']) + }) + + test('skips migration when only updatedAt is non-zero — updatedAt is not part of the guard', async () => { + // The guard deliberately ignores updatedAt: a non-zero timestamp with no + // domains or addresses does not represent meaningful phishing data. + const store = new PhishingIdbStorage(db) + const removeSpy = jest.fn(async () => {}) + + await store.ensureMigrated( + async () => ({ version: 0, updatedAt: 999, domains: [], addresses: [] }), + removeSpy + ) + + expect(await store.isEmpty()).toBe(true) + expect(removeSpy).not.toHaveBeenCalled() + }) + + test('an existing snapshot is fully preserved when migration is skipped', async () => { + // Covers the restart case: a second ensureMigrated on a populated store must + // neither overwrite nor merge the incoming legacy payload. + const store = new PhishingIdbStorage(db) + await store.saveSnapshot( + makeSnapshot({ version: 10, domains: ['kept.com', 'also-kept.com'] }) + ) + + await store.ensureMigrated( + async () => makeSnapshot({ version: 99, domains: ['should-not-appear.evil'] }), + async () => {} + ) + + const result = await store.loadSnapshot() + expect(result.version).toBe(10) + expect(result.domains).toEqual(['kept.com', 'also-kept.com']) + expect(result.domains).not.toContain('should-not-appear.evil') + }) + + test('concurrent ensureMigrated calls on the same connection converge', async () => { + // IDB-specific: two backend instances sharing one connection both observe + // isEmpty()=true before either writes. + const store1 = new PhishingIdbStorage(db) + const store2 = new PhishingIdbStorage(db) + const legacy = makeSnapshot({ version: 2, domains: ['concurrent.com'] }) + + await Promise.all([ + store1.ensureMigrated( + async () => legacy, + async () => {} + ), + store2.ensureMigrated( + async () => legacy, + async () => {} + ) + ]) + + const result = await store1.loadSnapshot() + expect(result.version).toBe(2) + expect(result.domains).toEqual(['concurrent.com']) + }) + + test('error from getStoredData propagates and leaves IDB unchanged', async () => { + const store = new PhishingIdbStorage(db) + + await expect( + store.ensureMigrated( + async () => { + throw new Error('storage unavailable') + }, + async () => {} + ) + ).rejects.toThrow('storage unavailable') + + expect(await store.isEmpty()).toBe(true) + }) + + test('error from removeStoredData propagates after IDB was already written', async () => { + const store = new PhishingIdbStorage(db) + const legacy = makeSnapshot({ version: 3 }) + + await expect( + store.ensureMigrated( + async () => legacy, + async () => { + throw new Error('remove failed') + } + ) + ).rejects.toThrow('remove failed') + + // IDB was written before removeStoredData was called + expect(await store.isEmpty()).toBe(false) + expect((await store.loadSnapshot()).version).toBe(3) + }) + }) +}) + +// ───────────────────────────────────────────────────────────────────────────── +// PhishingKeyValueStorage (chrome.storage.local / "static" backend) +// ───────────────────────────────────────────────────────────────────────────── + +describe('PhishingKeyValueStorage', () => { + describe('loadSnapshot', () => { + test('returns DEFAULT_PHISHING_SNAPSHOT when storage is empty', async () => { + const storage = makeStorageMock() + const backend = new PhishingKeyValueStorage(storage as any) + expect(await backend.loadSnapshot()).toEqual(DEFAULT_PHISHING_SNAPSHOT) + }) + + test('returns the snapshot already in storage', async () => { + const snap = makeSnapshot({ version: 2, domains: ['stored.com'] }) + const storage = makeStorageMock({ phishing: snap }) + const backend = new PhishingKeyValueStorage(storage as any) + expect(await backend.loadSnapshot()).toEqual(snap) + }) + + test('mutating the returned snapshot does not affect subsequent loads', async () => { + const storage = makeStorageMock({ phishing: makeSnapshot({ domains: ['original.com'] }) }) + const backend = new PhishingKeyValueStorage(storage as any) + + const result = await backend.loadSnapshot() + result.domains.push('injected.evil') + + const reloaded = await backend.loadSnapshot() + expect(reloaded.domains).toEqual(['original.com']) + expect(reloaded.domains).not.toContain('injected.evil') + }) + }) + + describe('saveSnapshot', () => { + test('persists data so a subsequent loadSnapshot returns it', async () => { + const storage = makeStorageMock() + const backend = new PhishingKeyValueStorage(storage as any) + const snap = makeSnapshot({ version: 3 }) + await backend.saveSnapshot(snap) + expect(await backend.loadSnapshot()).toEqual(snap) + }) + + test('overwrites the previous snapshot on a second save', async () => { + const storage = makeStorageMock() + const backend = new PhishingKeyValueStorage(storage as any) + await backend.saveSnapshot(makeSnapshot({ version: 1, domains: ['old.com'] })) + await backend.saveSnapshot(makeSnapshot({ version: 2, domains: ['new.com'] })) + const result = await backend.loadSnapshot() + expect(result.version).toBe(2) + expect(result.domains).toEqual(['new.com']) + }) + + test('mutating the snapshot object after saving does not affect subsequent loads', async () => { + const storage = makeStorageMock() + const backend = new PhishingKeyValueStorage(storage as any) + const snap = makeSnapshot({ domains: ['original.com'] }) + + await backend.saveSnapshot(snap) + // Mutate the caller's object after the save — storage must be isolated + snap.domains.push('injected-after-save.evil') + + const result = await backend.loadSnapshot() + expect(result.domains).toEqual(['original.com']) + expect(result.domains).not.toContain('injected-after-save.evil') + }) + }) + + describe('migration', () => { + test('is entirely inert — the data already lives in its final location', async () => { + const storage = makeStorageMock() + const backend = new PhishingKeyValueStorage(storage as any) + const getStoredSpy = jest.fn(async () => makeSnapshot()) + const removeSpy = jest.fn(async () => {}) + + // Never reports empty, so ensureMigrated can never decide to migrate + expect(await backend.isEmpty()).toBe(false) + + await backend.ensureMigrated(getStoredSpy, removeSpy) + expect(getStoredSpy).not.toHaveBeenCalled() + expect(removeSpy).not.toHaveBeenCalled() + + // An explicit import writes nothing either + await backend.migrateFromStorage(makeSnapshot()) + expect(await backend.loadSnapshot()).toEqual(DEFAULT_PHISHING_SNAPSHOT) + }) + }) +}) diff --git a/src/services/storage/phishingIdb.ts b/src/services/storage/phishingIdb.ts new file mode 100644 index 0000000000..5c899ee16a --- /dev/null +++ b/src/services/storage/phishingIdb.ts @@ -0,0 +1,156 @@ +import { IStorageController as IStorageControllerType } from '../../interfaces/storage' +import { AmbireIdbDatabase } from './idbDatabase' + +export interface PhishingSnapshot { + version: number + updatedAt: number + domains: string[] + addresses: string[] +} + +export const DEFAULT_PHISHING_SNAPSHOT: PhishingSnapshot = { + version: 0, + updatedAt: 0, + domains: [], + addresses: [] +} + +/** + * Persistence backend for the phishing list snapshot. + * Implementations: PhishingIdbStorage (IndexedDB) and PhishingKeyValueStorage (chrome.storage.local). + * PhishingController always holds one of these — there are no conditional IDB checks in the controller. + */ +export interface IPhishingOpsBackend { + /** + * Returns true if the store has no phishing snapshot yet. + * Used by ensureMigrated to decide whether migration is needed. + */ + isEmpty(): Promise + + /** + * Load the persisted phishing snapshot. + * Returns DEFAULT_PHISHING_SNAPSHOT if no data has been saved yet. + */ + loadSnapshot(): Promise + + /** + * Persist the full phishing snapshot (version + updatedAt + domains + addresses). + */ + saveSnapshot(data: PhishingSnapshot): Promise + + /** + * One-time migration from legacy chrome.storage.local to IDB. + * IDB backend: migrates if empty; key-value backend: no-op. + */ + ensureMigrated( + getStoredData: () => Promise, + removeStoredData: () => Promise + ): Promise + + /** + * Import a snapshot from the legacy storage format. + * Called by ensureMigrated; also exposed for tests. + */ + migrateFromStorage(data: PhishingSnapshot): Promise +} + +// ───────────────────────────────────────────────────────────────────────────── +// IDB backend +// ───────────────────────────────────────────────────────────────────────────── + +// The 'phishing' store holds a single document keyed by this constant. +// All reads and writes target this one record. +// +// NOTE: deliberately NOT in AMBIRE_IDB_SCHEMA. Adding it needs a dbVersion bump, and a +// shipped bump cannot be rolled back — not worth carrying for a store nothing reads. Add it +// in the same change that wires PhishingController. +const STORE_NAME = 'phishing' +const SNAPSHOT_KEY = 'snapshot' + +interface PhishingIdbRow extends PhishingSnapshot { + id: string // always SNAPSHOT_KEY +} + +export class PhishingIdbStorage implements IPhishingOpsBackend { + #db: AmbireIdbDatabase + #storeName = STORE_NAME + + constructor(db: AmbireIdbDatabase) { + this.#db = db + } + + async isEmpty(): Promise { + const count = await this.#db.count(this.#storeName) + return count === 0 + } + + async loadSnapshot(): Promise { + const row = (await this.#db.get(this.#storeName, SNAPSHOT_KEY)) as PhishingIdbRow | undefined + if (!row) return { ...DEFAULT_PHISHING_SNAPSHOT } + const { id: _id, ...snapshot } = row + return snapshot + } + + async saveSnapshot(data: PhishingSnapshot): Promise { + await this.#db.put(this.#storeName, { id: SNAPSHOT_KEY, ...data }) + } + + async migrateFromStorage(data: PhishingSnapshot): Promise { + await this.saveSnapshot(data) + } + + async ensureMigrated( + getStoredData: () => Promise, + removeStoredData: () => Promise + ): Promise { + const empty = await this.isEmpty() + if (!empty) return + + const stored = await getStoredData() + // Skip migration if storage also has no meaningful data. updatedAt is + // deliberately excluded: a timestamp with no domains or addresses does not + // represent meaningful phishing data. + if (!stored.version && !stored.domains.length && !stored.addresses.length) return + + await this.migrateFromStorage(stored) + await removeStoredData() + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Key-value storage backend (mobile / IDB-unavailable fallback) +// ───────────────────────────────────────────────────────────────────────────── + +/** + * chrome.storage.local–backed persistence for the phishing snapshot. + * Used in environments without IndexedDB support (mobile). + * Reads and writes the full snapshot blob directly — no IDB involved. + */ +export class PhishingKeyValueStorage implements IPhishingOpsBackend { + #storage: IStorageControllerType + + constructor(storage: IStorageControllerType) { + this.#storage = storage + } + + // Data is already in storage — isEmpty is meaningless for this backend. + async isEmpty(): Promise { + return false + } + + async loadSnapshot(): Promise { + return this.#storage.get('phishing', { ...DEFAULT_PHISHING_SNAPSHOT }) + } + + async saveSnapshot(data: PhishingSnapshot): Promise { + await this.#storage.set('phishing', data) + } + + // Migration is not needed — data is already in the right place. + async migrateFromStorage(_data: PhishingSnapshot): Promise {} + + async ensureMigrated( + _getStoredData: () => Promise, + _removeStoredData: () => Promise + ): Promise {} +}