fix: Fix purgeOnStart on aliased storages

docs/public-api/crawlee-types.api.md

@@ -570,7 +570,6 @@ export interface StorageBackend {
     createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
     createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
     getStorageBackendCacheKey?(): string;
-    // (undocumented)
     purge?(): Promise<void>;
     // (undocumented)
     stats?: {

packages/core/src/memory-storage/memory-storage.ts

@@ -7,6 +7,9 @@ import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
 
+/** The alias the default (unnamed) storage is opened under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+
 export interface MemoryStorageOptions {
     /**
      * Optional logger for MemoryStorageBackend warnings.
@@ -43,11 +46,14 @@ export class MemoryStorageBackend implements storage.StorageBackend {
         isAlias: boolean;
         cacheKey: string | undefined;
     } {
-        const isAlias = 'alias' in options && !!options.alias;
-        const rawKey = isAlias ? options.alias : (options.name ?? options.id);
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
+        // directly lands on the very storage the frontends would have opened.
+        const alias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        const rawKey = alias ?? options.name ?? options.id;
         // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { isAlias, cacheKey };
+        const cacheKey = rawKey === DEFAULT_STORAGE_ALIAS ? 'default' : rawKey;
+        return { isAlias: alias !== undefined, cacheKey };
     }
 
     async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {
@@ -147,34 +153,38 @@ export class MemoryStorageBackend implements storage.StorageBackend {
     }
 
     /**
-     * Cleans up the default storages before the run starts. For the in-memory storage this simply
-     * resets the in-memory state of the cached default dataset, key-value store and request queue.
+     * Cleans up the run-scoped storages before the run starts: the default one and every alias-keyed
+     * one. For the in-memory storage this simply resets the in-memory state of the cached backends.
      *
-     * As with `FileSystemStorageBackend`, the run's input (the `INPUT` key in the default key-value
-     * store) is preserved — only the rest of the default storages is cleared.
+     * Named storages are the opt-in way to keep data across runs, so they are left untouched. The run's
+     * input (the `INPUT` key in the default key-value store) is preserved as well, matching
+     * `FileSystemStorageBackend`.
      */
     async purge(): Promise<void> {
-        // The run default is opened via `{ alias: '__default__' }`, which `resolveStorageKey`
-        // normalizes to `cacheKey === 'default'` (with `name === undefined`) — that is the clause
-        // that actually matches it. The `name === 'default'` clause additionally covers a store a user
-        // explicitly opened via `{ name: 'default' }`. (`'__default__'` never reaches `cacheKey`,
-        // as it is always normalized to `'default'` first, so it does not need to be checked here.)
+        // Alias-keyed and default storages are unnamed (`resolveStorageKey` only sets `name` for named
+        // ones), which is what marks them as belonging to a single run — same rule as crawlee-python's
+        // `_purge_if_needed`. The extra `name === 'default'` clause covers a store opened via
+        // `{ name: 'default' }`: that collapses onto the same `cacheKey` as the default storage, so the
+        // cached backend the run's default resolves to may well carry that name.
+        const isRunScoped = (store: { name?: string }) => store.name === undefined || store.name === 'default';
+
         const isDefault = (store: { name?: string; cacheKey: string }) =>
             store.name === 'default' || store.cacheKey === 'default';
 
-        const purgeDefaults = async <T extends { name?: string; cacheKey: string }>(
+        const purgeRunScoped = async <T extends { name?: string; cacheKey: string }>(
             cache: T[],
             purgeStore: (store: T) => Promise<void>,
         ) => {
-            await Promise.all(cache.filter(isDefault).map(async (store) => purgeStore(store)));
+            await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
         };
 
         await Promise.all([
-            // Preserve the run input (INPUT) when purging the default key-value store, matching
-            // `FileSystemStorageBackend`.
-            purgeDefaults(this.keyValueStoreBackendCache, async (store) => store.purgeExceptInput()),
-            purgeDefaults(this.datasetBackendCache, async (store) => store.purge()),
-            purgeDefaults(this.requestQueueBackendCache, async (store) => store.purge()),
+            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+            purgeRunScoped(this.keyValueStoreBackendCache, async (store) =>
+                isDefault(store) ? store.purgeExceptInput() : store.purge(),
+            ),
+            purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),
         ]);
     }
 

packages/core/src/storages/throttling_request_manager.ts

@@ -101,7 +101,7 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
      *
      * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
      * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
-     * their queue, so re-running the crawl without purging storages picks them up once the domain recovers.
+     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
      *
      * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
      * whole point there.
@@ -191,9 +191,9 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
     private readonly log: CrawleeLogger;
 
     /**
-     * Sub-managers are keyed by a stable alias, so they outlive the process. They must therefore be reopened
-     * for every configured domain rather than created on first insert - otherwise a restart sees an empty map,
-     * reports the crawl finished, and strands whatever the previous run left in them.
+     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
+     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
+     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
      */
     private subManagersReady?: Promise<void>;
 

packages/core/src/storages/utils.ts

@@ -21,7 +21,8 @@ interface PurgeDefaultStorageOptions {
 
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
@@ -35,7 +36,8 @@ interface PurgeDefaultStorageOptions {
 export async function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}

packages/fs-storage/src/file-system-storage.ts

@@ -14,6 +14,12 @@ import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
 
+/** The alias `@crawlee/core` opens the default (unnamed) storage under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+
+/** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
+const DEFAULT_STORAGE_DIRECTORY = 'default';
+
 export interface FileSystemStorageOptions {
     /**
      * Path to directory where the data will be saved.
@@ -93,11 +99,16 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
         alias?: string;
         cacheKey: string | undefined;
     } {
-        const isAlias = 'alias' in options && !!options.alias;
-        const rawKey = isAlias ? options.alias : (options.name ?? options.id);
-        // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { id: options.id, name: options.name, alias: options.alias, cacheKey };
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in @crawlee/core, so that a backend used directly
+        // lands on the very storage the frontends would have opened.
+        const requestedAlias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `__default__` is an internal sentinel and must not escape onto disk: the default storage lives
+        // in `default`, which is what the docs, the project templates and every pre-existing local
+        // `storage/` directory expect. Normalizing here keeps the cache key and the directory in step.
+        const alias = requestedAlias === DEFAULT_STORAGE_ALIAS ? DEFAULT_STORAGE_DIRECTORY : requestedAlias;
+        const cacheKey = alias ?? options.name ?? options.id;
+        return { id: options.id, name: options.name, alias, cacheKey };
     }
 
     async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {
@@ -242,7 +253,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
     ): Promise<string | undefined> {
         // Directory named exactly after the string: return its real (metadata) id, which may differ
         // from the string when the string is a name rather than an id.
-        const directId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, entryNameOrId));
+        const directId = (await FileSystemStorageBackend.readMetadata(resolve(baseDirectory, entryNameOrId)))?.id;
         if (directId !== undefined) {
             return directId;
         }
@@ -260,7 +271,8 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
                 continue;
             }
 
-            const metadataId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, directory.name));
+            const metadataId = (await FileSystemStorageBackend.readMetadata(resolve(baseDirectory, directory.name)))
+                ?.id;
             if (metadataId === entryNameOrId) {
                 return metadataId;
             }
@@ -269,44 +281,116 @@ export class FileSystemStorageBackend implements storage.StorageBackend {
         return undefined;
     }
 
-    /** Read the `id` field from a storage directory's `__metadata__.json`, or `undefined` if absent. */
-    private static async readMetadataId(storageDirectory: string): Promise<string | undefined> {
+    /** Read a storage directory's `__metadata__.json`, or `undefined` if there is none to read. */
+    private static async readMetadata(
+        storageDirectory: string,
+    ): Promise<{ id?: string; name?: string | null } | undefined> {
         try {
             const fileContent = await readFile(resolve(storageDirectory, '__metadata__.json'), 'utf8');
-            return (JSON.parse(fileContent) as { id?: string }).id;
+            return JSON.parse(fileContent) as { id?: string; name?: string | null };
         } catch {
-            // Directory missing, or no/unreadable metadata file — no id to report.
+            // Directory missing, or no/unreadable metadata file — nothing to report.
             return undefined;
         }
     }
 
     /**
-     * Cleans up the default storages before the run starts:
-     *  - the default dataset;
-     *  - all records from the default key-value store, except for the "INPUT" key;
-     *  - the default request queue.
+     * Cleans up the storages that belong to a single run before the run starts:
+     *  - the default dataset and request queue, plus every alias-keyed one;
+     *  - all records from the default key-value store except for the "INPUT" key, and all records from
+     *    every alias-keyed key-value store.
+     *
+     * Named storages are the opt-in way of keeping data across runs, so they are left untouched.
      */
     async purge(): Promise<void> {
-        // Resolve the default stores up front so leftover on-disk records are purged even when the
-        // store has not been opened in this process yet (e.g. a fresh run over a pre-existing
-        // directory). Opening caches the backend, so the subsequent purge operates on a real backend.
-        // The default store is opened via the internal `__default__` alias (see resolveStorageIdentifier
-        // in @crawlee/core), which resolves to the `default` cache key — match that here so we purge the
-        // very backend the default open would return rather than creating a divergent one.
-        const [defaultKeyValueStore, defaultDataset, defaultRequestQueue] = await Promise.all([
-            this.createKeyValueStoreBackend({ alias: '__default__' }) as Promise<KeyValueStoreBackend>,
-            this.createDatasetBackend({ alias: '__default__' }) as Promise<DatasetBackend>,
-            this.createRequestQueueBackend({ alias: '__default__' }) as Promise<RequestQueueBackend>,
-        ]);
-
         await Promise.all([
-            // Preserve the run input (INPUT) when purging the default key-value store.
-            defaultKeyValueStore.purgeExceptInput(),
-            defaultDataset.purge(),
-            defaultRequestQueue.purge(),
+            this.purgeRunScopedStorages(
+                this.keyValueStoresDirectory,
+                async (alias) => this.createKeyValueStoreBackend({ alias }) as Promise<KeyValueStoreBackend>,
+                // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+                async (store, isDefault) => (isDefault ? store.purgeExceptInput() : store.purge()),
+            ),
+            this.purgeRunScopedStorages(
+                this.datasetsDirectory,
+                async (alias) => this.createDatasetBackend({ alias }) as Promise<DatasetBackend>,
+                async (store) => store.purge(),
+            ),
+            this.purgeRunScopedStorages(
+                this.requestQueuesDirectory,
+                async (alias) => this.createRequestQueueBackend({ alias }) as Promise<RequestQueueBackend>,
+                async (store) => store.purge(),
+            ),
         ]);
     }
 
+    /**
+     * Purge every run-scoped storage under `storagesDirectory` — the default one and every alias-keyed
+     * one, whether or not it has been opened in this process yet.
+     *
+     * Storages are opened through `open` rather than purged on disk directly, both so that a leftover
+     * from a previous process is reachable at all, and so that a storage already open in this process is
+     * purged through the very backend the run is using instead of a divergent second one.
+     */
+    private async purgeRunScopedStorages<T>(
+        storagesDirectory: string,
+        open: (alias: string) => Promise<T>,
+        purgeStorage: (storage: T, isDefault: boolean) => Promise<void>,
+    ): Promise<void> {
+        // The default storage is listed unconditionally, so that a run over an empty directory still ends
+        // up with it opened (and cached) exactly as it was before. Deduplicating by cache key then keeps
+        // it to a single open: every run after the first also finds its `default` directory on disk, and
+        // opening the same storage twice concurrently would race two backends onto one directory.
+        const aliasesByCacheKey = new Map<string | undefined, string>();
+
+        for (const alias of [
+            DEFAULT_STORAGE_ALIAS,
+            ...(await FileSystemStorageBackend.listUnnamedStorages(storagesDirectory)),
+        ]) {
+            const { cacheKey } = FileSystemStorageBackend.resolveStorageKey({ alias });
+            if (!aliasesByCacheKey.has(cacheKey)) {
+                aliasesByCacheKey.set(cacheKey, alias);
+            }
+        }
+
+        await Promise.all(
+            Array.from(aliasesByCacheKey, async ([cacheKey, alias]) => {
+                await purgeStorage(await open(alias), cacheKey === DEFAULT_STORAGE_DIRECTORY);
+            }),
+        );
+    }
+
+    /**
+     * The directory names of the on-disk storages under `storagesDirectory` that Crawlee created without
+     * a name — the default storage and every alias-keyed one. Since the directory is named after the
+     * storage's `name ?? alias ?? id`, the name is read from the metadata rather than guessed.
+     *
+     * A directory with no readable `__metadata__.json` was not created by Crawlee (a hand-placed input
+     * directory, say), so it is left out — purging it would destroy data we never wrote.
+     */
+    private static async listUnnamedStorages(storagesDirectory: string): Promise<string[]> {
+        let directories;
+        try {
+            directories = await opendir(storagesDirectory);
+        } catch {
+            return [];
+        }
+
+        const unnamed: string[] = [];
+
+        for await (const directory of directories) {
+            if (!directory.isDirectory()) {
+                continue;
+            }
+
+            const metadata = await FileSystemStorageBackend.readMetadata(resolve(storagesDirectory, directory.name));
+            if (metadata !== undefined && typeof metadata.name !== 'string') {
+                unnamed.push(directory.name);
+            }
+        }
+
+        return unnamed;
+    }
+
     /**
      * This method should be called at the end of the process, to ensure all data is saved.
      *