fix(crawler): treat HTTP 429 as rate-limit condition with cooldown backoff

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -47,6 +47,7 @@ import {
     RequestQueue,
     RequestQueueV1,
     RequestState,
+    RateLimitError,
     RetryRequestError,
     Router,
     SessionError,
@@ -55,7 +56,7 @@ import {
     validators,
 } from '@crawlee/core';
 import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';
-import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
+import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS, sleep } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
 import { ensureDir, writeFile, writeJSON } from 'fs-extra';
 import ow, { ArgumentError } from 'ow';
@@ -263,6 +264,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     sameDomainDelaySecs?: number;
 
+    /**
+     * How long to wait before retrying a request that failed with a rate limit error (HTTP 429).
+     * This value will only be used if the server does not return a `Retry-After` header.
+     * @default 60
+     */
+    rateLimitCooldownSecs?: number;
+
     /**
      * Maximum number of session rotations per request.
      * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
@@ -547,6 +555,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
     protected maxRequestRetries: number;
     protected maxCrawlDepth?: number;
     protected sameDomainDelayMillis: number;
+    protected rateLimitCooldownMillis: number;
     protected domainAccessedTime: Map<string, number>;
     protected maxSessionRotations: number;
     protected maxRequestsPerCrawl?: number;
@@ -598,6 +607,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         handleFailedRequestFunction: ow.optional.function,
         maxRequestRetries: ow.optional.number,
         sameDomainDelaySecs: ow.optional.number,
+        rateLimitCooldownSecs: ow.optional.number,
         maxSessionRotations: ow.optional.number,
         maxRequestsPerCrawl: ow.optional.number,
         maxCrawlDepth: ow.optional.number,
@@ -641,6 +651,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             requestManager,
             maxRequestRetries = 3,
             sameDomainDelaySecs = 0,
+            rateLimitCooldownSecs = 60,
             maxSessionRotations = 10,
             maxRequestsPerCrawl,
             maxCrawlDepth,
@@ -768,6 +779,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         this.maxRequestRetries = maxRequestRetries;
         this.maxCrawlDepth = maxCrawlDepth;
         this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+        this.rateLimitCooldownMillis = rateLimitCooldownSecs * 1000;
         this.maxSessionRotations = maxSessionRotations;
         this.stats = new Statistics({
             logMessage: `${log.getOptions().prefix} request statistics:`,
@@ -1674,7 +1686,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 throw secondaryError;
             }
             // decrease the session score if the request fails (but the error handler did not throw)
-            crawlingContext.session?.markBad();
+            if (!(err instanceof RateLimitError)) {
+                crawlingContext.session?.markBad();
+            }
         } finally {
             await this._cleanupContext(crawlingContext);
 
@@ -1850,6 +1864,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                     retryCount,
                 });
 
+                if (error instanceof RateLimitError) {
+                    const delayMillis = error.delayMillis || this.rateLimitCooldownMillis;
+                    await sleep(delayMillis);
+                    await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+                    return;
+                }
+
                 await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
                 return;
             }

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -29,6 +29,7 @@ import {
     tryAbsoluteURL,
     validators,
 } from '@crawlee/basic';
+import { RateLimitError } from '@crawlee/core';
 import type {
     BrowserController,
     BrowserPlugin,
@@ -726,7 +727,28 @@ export abstract class BrowserCrawler<
 
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                const status = response.status();
+                if (status === 429) {
+                    let delayMillis = this.rateLimitCooldownMillis;
+                    const retryAfterHeader =
+                        typeof response.headers === 'function' ? response.headers()['retry-after'] : undefined;
+
+                    if (retryAfterHeader) {
+                        const parsedSeconds = parseInt(retryAfterHeader, 10);
+                        if (!Number.isNaN(parsedSeconds)) {
+                            delayMillis = parsedSeconds * 1000;
+                        } else {
+                            const parsedDate = Date.parse(retryAfterHeader);
+                            if (!Number.isNaN(parsedDate)) {
+                                delayMillis = Math.max(0, parsedDate - Date.now());
+                            }
+                        }
+                    }
+
+                    throw new RateLimitError(undefined, delayMillis);
+                }
+
+                this._throwOnBlockedRequest(session, status);
             } else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
             }

packages/core/src/errors.ts

@@ -35,3 +35,16 @@ export class SessionError extends RetryRequestError {
         super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
     }
 }
+
+/**
+ * Errors of `RateLimitError` type will trigger a delay before retrying the request.
+ */
+export class RateLimitError extends RetryRequestError {
+    readonly delayMillis?: number;
+
+    constructor(message?: string, delayMillis?: number) {
+        super(message ?? 'Request is being retried due to rate limit');
+        this.delayMillis = delayMillis;
+        Object.setPrototypeOf(this, RateLimitError.prototype);
+    }
+}

packages/core/src/session_pool/consts.ts

@@ -1,3 +1,3 @@
-export const BLOCKED_STATUS_CODES = [401, 403, 429];
+export const BLOCKED_STATUS_CODES = [401, 403];
 export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;

packages/core/src/session_pool/session_pool.ts

@@ -56,7 +56,7 @@ export interface SessionPoolOptions {
     /**
      * Specifies which response status codes are considered as blocked.
      * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
+     * @default [401, 403]
      */
     blockedStatusCodes?: number[];
 

packages/http-crawler/src/internals/http-crawler.ts

@@ -29,6 +29,7 @@ import {
     SessionError,
     validators,
 } from '@crawlee/basic';
+import { RateLimitError } from '@crawlee/core';
 import type { HttpResponse, StreamingHttpResponse } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';
@@ -540,6 +541,25 @@ export class HttpCrawler<
                 return $;
             };
 
+            if (response.statusCode === 429) {
+                const retryAfterHeader = response.headers['retry-after'];
+                let delayMillis = this.rateLimitCooldownMillis;
+
+                if (retryAfterHeader) {
+                    const parsedSeconds = parseInt(retryAfterHeader, 10);
+                    if (!Number.isNaN(parsedSeconds)) {
+                        delayMillis = parsedSeconds * 1000;
+                    } else {
+                        const parsedDate = Date.parse(retryAfterHeader);
+                        if (!Number.isNaN(parsedDate)) {
+                            delayMillis = Math.max(0, parsedDate - Date.now());
+                        }
+                    }
+                }
+
+                throw new RateLimitError(undefined, delayMillis);
+            }
+
             if (this.useSessionPool) {
                 this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!);
             }
@@ -915,7 +935,8 @@ export class HttpCrawler<
         // eslint-disable-next-line dot-notation -- accessing private property
         const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
         // if we retry the request, can the Content-Type change?
-        const isTransientContentType = statusCode! >= 500 || blockedStatusCodes.includes(statusCode!);
+        const isTransientContentType =
+            statusCode! >= 500 || statusCode === 429 || blockedStatusCodes.includes(statusCode!);
 
         if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;

test/core/crawlers/browser_crawler.test.ts

@@ -615,7 +615,7 @@ describe('BrowserCrawler', () => {
 
             await crawler.run();
 
-            expect(failedRequests.length).toBe(3);
+            expect(failedRequests.length).toBe(BLOCKED_STATUS_CODES.length);
             failedRequests.forEach((fr) => {
                 const [msg] = fr.errorMessages;
                 expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
@@ -750,7 +750,7 @@ describe('BrowserCrawler', () => {
 
             await crawler.run();
 
-            expect(failedRequests.length).toBe(3);
+            expect(failedRequests.length).toBe(BLOCKED_STATUS_CODES.length);
             failedRequests.forEach((fr) => {
                 const [msg] = fr.errorMessages;
                 expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
@@ -807,6 +807,55 @@ describe('BrowserCrawler', () => {
         }
     });
 
+    test.concurrent('should handle 429 Rate Limit with Retry-After header', async () => {
+        const localStorageEmulator = new MemoryStorageEmulator();
+        await localStorageEmulator.init();
+        const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+        try {
+            const succeeded: Request[] = [];
+            const crawler = new BrowserCrawlerTest({
+                browserPoolOptions: {
+                    browserPlugins: [puppeteerPlugin],
+                },
+                useSessionPool: true,
+                sessionPoolOptions: {
+                    maxPoolSize: 1,
+                },
+                maxConcurrency: 1,
+                maxRequestRetries: 1,
+                requestHandler: async ({ request }) => {
+                    succeeded.push(request);
+                },
+            });
+
+            // @ts-expect-error Overriding protected method
+            crawler._navigationHandler = async ({ request }) => {
+                if (request.retryCount === 0) {
+                    return {
+                        status: () => 429,
+                        headers: () => ({ 'retry-after': '1' }),
+                    };
+                }
+
+                return {
+                    status: () => 200,
+                    headers: () => ({}),
+                };
+            };
+
+            const start = Date.now();
+            await crawler.run([serverAddress]);
+            const end = Date.now();
+
+            expect(succeeded).toHaveLength(1);
+            expect(succeeded[0].retryCount).toBe(1);
+            expect(end - start).toBeGreaterThanOrEqual(1000);
+        } finally {
+            await localStorageEmulator.destroy();
+        }
+    });
+
     test.concurrent('should increment session usage correctly', async () => {
         const localStorageEmulator = new MemoryStorageEmulator();
         await localStorageEmulator.init();

test/core/crawlers/cheerio_crawler.test.ts

@@ -933,7 +933,7 @@ describe('CheerioCrawler', () => {
         });
 
         test('should retire session on "blocked" status codes', async () => {
-            for (const code of [401, 403, 429]) {
+            for (const code of [401, 403]) {
                 const failed: Request[] = [];
                 const sessions: Session[] = [];
                 const crawler = new CheerioCrawler({

test/core/crawlers/http_crawler.test.ts

@@ -61,6 +61,17 @@ router.set('/403-with-octet-stream', (req, res) => {
     res.end();
 });
 
+router.set('/429-rate-limit', (req, res) => {
+    res.statusCode = 429;
+    res.setHeader('Retry-After', '1'); // 1 second
+    res.end();
+});
+
+router.set('/429-rate-limit-no-header', (req, res) => {
+    res.statusCode = 429;
+    res.end();
+});
+
 let server: http.Server;
 let url: string;
 
@@ -397,6 +408,81 @@ describe.each(
         expect(succeeded[0].retryCount).toBe(1);
     });
 
+    test('should handle 429 Rate Limit with Retry-After header', async () => {
+        const succeeded: any[] = [];
+        const sessionIds: string[] = [];
+        const crawler = new HttpCrawler({
+            httpClient,
+            maxConcurrency: 1,
+            maxRequestRetries: 1,
+            sessionPoolOptions: {
+                maxPoolSize: 1,
+            },
+            preNavigationHooks: [
+                async ({ request, session }, gotOptions) => {
+                    sessionIds.push(session!.id);
+                    if (request.retryCount === 0) {
+                        request.url = `${url}/429-rate-limit`;
+                    } else {
+                        request.url = url;
+                    }
+                    if (gotOptions) {
+                        gotOptions.throwHttpErrors = false;
+                    }
+                },
+            ],
+            requestHandler: async ({ request }) => {
+                succeeded.push(request);
+            },
+            failedRequestHandler: async ({ request, error }) => {
+                console.error('FAILED', request.retryCount, request.url, error.message);
+            },
+        });
+
+        const start = Date.now();
+        await crawler.run([url]);
+        const end = Date.now();
+
+        expect(succeeded).toHaveLength(1);
+        expect(succeeded[0].retryCount).toBe(1);
+        expect(sessionIds).toHaveLength(2);
+        expect(sessionIds[0]).toBe(sessionIds[1]);
+        expect(end - start).toBeGreaterThanOrEqual(1000); // Should delay for at least 1s
+    });
+
+    test('should handle 429 Rate Limit without Retry-After header via cooldown', async () => {
+        const succeeded: any[] = [];
+        const crawler = new HttpCrawler({
+            httpClient,
+            maxConcurrency: 1,
+            maxRequestRetries: 1,
+            rateLimitCooldownSecs: 1,
+            preNavigationHooks: [
+                async ({ request }, gotOptions) => {
+                    if (request.retryCount === 0) {
+                        request.url = `${url}/429-rate-limit-no-header`;
+                    } else {
+                        request.url = url;
+                    }
+                    if (gotOptions) {
+                        gotOptions.throwHttpErrors = false;
+                    }
+                },
+            ],
+            requestHandler: async ({ request }) => {
+                succeeded.push(request);
+            },
+        });
+
+        const start = Date.now();
+        await crawler.run([url]);
+        const end = Date.now();
+
+        expect(succeeded).toHaveLength(1);
+        expect(succeeded[0].retryCount).toBe(1);
+        expect(end - start).toBeGreaterThanOrEqual(1000);
+    });
+
     test.skipIf(httpClient instanceof ImpitHttpClient)('should work with cacheable-request', async () => {
         const isFromCache: Record<string, boolean> = {};
         const cache = new Map();