fix(crawler): treat HTTP 429 as rate-limit condition with cooldown backoff

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -47,6 +47,7 @@ import {
     RequestQueue,
     RequestQueueV1,
     RequestState,
+    RateLimitError,
     RetryRequestError,
     Router,
     SessionError,
@@ -55,7 +56,7 @@ import {
     validators,
 } from '@crawlee/core';
 import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';
-import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
+import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS, sleep } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
 import { ensureDir, writeFile, writeJSON } from 'fs-extra';
 import ow, { ArgumentError } from 'ow';
@@ -263,6 +264,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     sameDomainDelaySecs?: number;
 
+    /**
+     * How long to wait before retrying a request that failed with a rate limit error (HTTP 429).
+     * This value will only be used if the server does not return a `Retry-After` header.
+     * @default 0
+     */
+    rateLimitCooldownSecs?: number;
+
     /**
      * Maximum number of session rotations per request.
      * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
@@ -547,6 +555,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
     protected maxRequestRetries: number;
     protected maxCrawlDepth?: number;
     protected sameDomainDelayMillis: number;
+    protected rateLimitCooldownMillis: number;
     protected domainAccessedTime: Map<string, number>;
     protected maxSessionRotations: number;
     protected maxRequestsPerCrawl?: number;
@@ -598,6 +607,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         handleFailedRequestFunction: ow.optional.function,
         maxRequestRetries: ow.optional.number,
         sameDomainDelaySecs: ow.optional.number,
+        rateLimitCooldownSecs: ow.optional.number,
         maxSessionRotations: ow.optional.number,
         maxRequestsPerCrawl: ow.optional.number,
         maxCrawlDepth: ow.optional.number,
@@ -641,6 +651,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             requestManager,
             maxRequestRetries = 3,
             sameDomainDelaySecs = 0,
+            rateLimitCooldownSecs = 0,
             maxSessionRotations = 10,
             maxRequestsPerCrawl,
             maxCrawlDepth,
@@ -768,6 +779,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         this.maxRequestRetries = maxRequestRetries;
         this.maxCrawlDepth = maxCrawlDepth;
         this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+        this.rateLimitCooldownMillis = rateLimitCooldownSecs * 1000;
         this.maxSessionRotations = maxSessionRotations;
         this.stats = new Statistics({
             logMessage: `${log.getOptions().prefix} request statistics:`,
@@ -874,7 +886,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             log,
         };
 
-        this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
+        this.autoscaledPoolOptions = {
+            ...autoscaledPoolOptions,
+            ...basicCrawlerAutoscaledPoolConfiguration,
+        };
     }
 
     /**
@@ -921,7 +936,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
     private getPeriodicLogger() {
         let previousState = { ...this.stats.state };
 
-        const getOperationMode = (): { mode: 'ERROR' | 'REGULAR'; failedDelta: number } => {
+        const getOperationMode = (): {
+            mode: 'ERROR' | 'REGULAR';
+            failedDelta: number;
+        } => {
             const { requestsFailed } = this.stats.state;
             const { requestsFailed: previousRequestsFailed } = previousState;
 
@@ -1253,15 +1271,24 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             await Promise.all(
                 [...skippedBecauseOfRobots]
                     .map((url) => {
-                        return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
+                        return this.handleSkippedRequest({
+                            url,
+                            reason: 'robotsTxt',
+                        });
                     })
                     .concat(
                         skippedBecauseOfLimit.map((request) => {
                             const url = typeof request === 'string' ? request : request.url!;
-                            return this.handleSkippedRequest({ url, reason: 'limit' });
+                            return this.handleSkippedRequest({
+                                url,
+                                reason: 'limit',
+                            });
                         }),
                         [...skippedBecauseOfMaxCrawlDepth].map((url) => {
-                            return this.handleSkippedRequest({ url, reason: 'depth' });
+                            return this.handleSkippedRequest({
+                                url,
+                                reason: 'depth',
+                            });
                         }),
                     ),
             );
@@ -1533,7 +1560,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 source['inProgress'].add(request.id!);
             }
 
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            await source.reclaimRequest(request, {
+                forefront: request.userData?.__crawlee?.forefront,
+            });
         }, delay);
 
         return true;
@@ -1673,8 +1702,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 request.state = RequestState.ERROR;
                 throw secondaryError;
             }
-            // decrease the session score if the request fails (but the error handler did not throw)
-            crawlingContext.session?.markBad();
+
+            if (!(err instanceof RateLimitError)) {
+                crawlingContext.session?.markBad();
+            }
+
+            if (err instanceof RateLimitError) {
+                const delayMillis = (err as RateLimitError).delayMillis ?? this.rateLimitCooldownMillis;
+                if (delayMillis > 0) {
+                    this.log.debug(`Waiting ${delayMillis}ms before next attempt due to rate limiting (Retry-After).`);
+                    await sleep(delayMillis);
+                }
+            }
         } finally {
             await this._cleanupContext(crawlingContext);
 
@@ -1850,7 +1889,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                     retryCount,
                 });
 
-                await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+                if (error instanceof RateLimitError) {
+                    // Reclaim immediately; the rate-limit sleep is applied in _runTaskFunction
+                    // *outside* the internalTimeoutMillis wrapper to avoid timeout conflicts.
+                    await source.reclaimRequest(request, {
+                        forefront: request.userData?.__crawlee?.forefront,
+                    });
+                    return;
+                }
+
+                await source.reclaimRequest(request, {
+                    forefront: request.userData?.__crawlee?.forefront,
+                });
                 return;
             }
         }
@@ -1878,7 +1928,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         try {
             return (await cb()) as T;
         } catch (e: any) {
-            Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
+            Object.defineProperty(e, 'triggeredFromUserHandler', {
+                value: true,
+            });
             throw e;
         }
     }
@@ -2074,7 +2126,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 return baseUrl.hostname === loadedBaseUrl.hostname;
             }
             case EnqueueStrategy.SameDomain: {
-                const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
+                const baseUrlHostname = getDomain(baseUrl.hostname, {
+                    mixedInputs: false,
+                });
 
                 if (baseUrlHostname) {
                     const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -29,6 +29,7 @@ import {
     tryAbsoluteURL,
     validators,
 } from '@crawlee/basic';
+import { RateLimitError } from '@crawlee/core';
 import type {
     BrowserController,
     BrowserPlugin,
@@ -726,7 +727,28 @@ export abstract class BrowserCrawler<
 
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                const status = response.status();
+                if (status === 429) {
+                    let delayMillis = this.rateLimitCooldownMillis;
+                    const retryAfterHeader =
+                        typeof response.headers === 'function' ? response.headers()['retry-after'] : undefined;
+
+                    if (retryAfterHeader) {
+                        const parsedSeconds = parseInt(retryAfterHeader, 10);
+                        if (!Number.isNaN(parsedSeconds)) {
+                            delayMillis = parsedSeconds * 1000;
+                        } else {
+                            const parsedDate = Date.parse(retryAfterHeader);
+                            if (!Number.isNaN(parsedDate)) {
+                                delayMillis = Math.max(0, parsedDate - Date.now());
+                            }
+                        }
+                    }
+
+                    throw new RateLimitError(undefined, delayMillis);
+                }
+
+                this._throwOnBlockedRequest(session, status);
             } else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
             }

packages/core/src/errors.ts

@@ -35,3 +35,16 @@ export class SessionError extends RetryRequestError {
         super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
     }
 }
+
+/**
+ * Errors of `RateLimitError` type will trigger a delay before retrying the request.
+ */
+export class RateLimitError extends RetryRequestError {
+    readonly delayMillis?: number;
+
+    constructor(message?: string, delayMillis?: number) {
+        super(message ?? 'Request is being retried due to rate limit');
+        this.delayMillis = delayMillis;
+        Object.setPrototypeOf(this, RateLimitError.prototype);
+    }
+}

packages/core/src/session_pool/consts.ts

@@ -1,3 +1,3 @@
-export const BLOCKED_STATUS_CODES = [401, 403, 429];
+export const BLOCKED_STATUS_CODES = [401, 403];
 export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;

packages/core/src/session_pool/session_pool.ts

@@ -56,7 +56,7 @@ export interface SessionPoolOptions {
     /**
      * Specifies which response status codes are considered as blocked.
      * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
+     * @default [401, 403]
      */
     blockedStatusCodes?: number[];
 

packages/http-crawler/src/internals/http-crawler.ts

@@ -29,6 +29,7 @@ import {
     SessionError,
     validators,
 } from '@crawlee/basic';
+import { RateLimitError } from '@crawlee/core';
 import type { HttpResponse, StreamingHttpResponse } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';
@@ -540,6 +541,25 @@ export class HttpCrawler<
                 return $;
             };
 
+            if (response.statusCode === 429) {
+                const retryAfterHeader = response.headers['retry-after'];
+                let delayMillis = this.rateLimitCooldownMillis;
+
+                if (retryAfterHeader) {
+                    const parsedSeconds = parseInt(retryAfterHeader, 10);
+                    if (!Number.isNaN(parsedSeconds)) {
+                        delayMillis = parsedSeconds * 1000;
+                    } else {
+                        const parsedDate = Date.parse(retryAfterHeader);
+                        if (!Number.isNaN(parsedDate)) {
+                            delayMillis = Math.max(0, parsedDate - Date.now());
+                        }
+                    }
+                }
+
+                throw new RateLimitError(undefined, delayMillis);
+            }
+
             if (this.useSessionPool) {
                 this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!);
             }
@@ -915,7 +935,8 @@ export class HttpCrawler<
         // eslint-disable-next-line dot-notation -- accessing private property
         const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
         // if we retry the request, can the Content-Type change?
-        const isTransientContentType = statusCode! >= 500 || blockedStatusCodes.includes(statusCode!);
+        const isTransientContentType =
+            statusCode! >= 500 || statusCode === 429 || blockedStatusCodes.includes(statusCode!);
 
         if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;

test/core/crawlers/browser_crawler.test.ts

@@ -615,7 +615,7 @@ describe('BrowserCrawler', () => {
 
             await crawler.run();
 
-            expect(failedRequests.length).toBe(3);
+            expect(failedRequests.length).toBe(BLOCKED_STATUS_CODES.length);
             failedRequests.forEach((fr) => {
                 const [msg] = fr.errorMessages;
                 expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
@@ -750,7 +750,7 @@ describe('BrowserCrawler', () => {
 
             await crawler.run();
 
-            expect(failedRequests.length).toBe(3);
+            expect(failedRequests.length).toBe(BLOCKED_STATUS_CODES.length);
             failedRequests.forEach((fr) => {
                 const [msg] = fr.errorMessages;
                 expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
@@ -807,6 +807,55 @@ describe('BrowserCrawler', () => {
         }
     });
 
+    test.concurrent('should handle 429 Rate Limit with Retry-After header', async () => {
+        const localStorageEmulator = new MemoryStorageEmulator();
+        await localStorageEmulator.init();
+        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+        try {
+            const succeeded: Request[] = [];
+            const crawler = new BrowserCrawlerTest({
+                browserPoolOptions: {
+                    browserPlugins: [puppeteerPlugin],
+                },
+                useSessionPool: true,
+                sessionPoolOptions: {
+                    maxPoolSize: 1,
+                },
+                maxConcurrency: 1,
+                maxRequestRetries: 1,
+                requestHandler: async ({ request }) => {
+                    succeeded.push(request);
+                },
+            });
+
+            // @ts-expect-error Overriding protected method
+            crawler._navigationHandler = async ({ request }) => {
+                if (request.retryCount === 0) {
+                    return {
+                        status: () => 429,
+                        headers: () => ({ 'retry-after': '1' }),
+                    };
+                }
+
+                return {
+                    status: () => 200,
+                    headers: () => ({}),
+                };
+            };
+
+            const start = Date.now();
+            await crawler.run([serverAddress]);
+            const end = Date.now();
+
+            expect(succeeded).toHaveLength(1);
+            expect(succeeded[0].retryCount).toBe(1);
+            expect(end - start).toBeGreaterThanOrEqual(1000);
+        } finally {
+            await localStorageEmulator.destroy();
+        }
+    });
+
     test.concurrent('should increment session usage correctly', async () => {
         const localStorageEmulator = new MemoryStorageEmulator();
         await localStorageEmulator.init();

test/core/crawlers/cheerio_crawler.test.ts

@@ -933,7 +933,7 @@ describe('CheerioCrawler', () => {
         });
 
         test('should retire session on "blocked" status codes', async () => {
-            for (const code of [401, 403, 429]) {
+            for (const code of [401, 403]) {
                 const failed: Request[] = [];
                 const sessions: Session[] = [];
                 const crawler = new CheerioCrawler({