apify · janbuchar · Aug 10, 2026 · Jun 16, 2026 · Aug 5, 2026 · Aug 5, 2026
diff --git a/docs/guides/request_loaders.mdx b/docs/guides/request_loaders.mdx
@@ -26,6 +26,7 @@ The request loader abstractions are built around two interfaces and a couple of
 - <ApiLink to="core/interface/IRequestLoader">`IRequestLoader`</ApiLink>: The base interface for reading requests in a crawl.
 - <ApiLink to="core/interface/IRequestManager">`IRequestManager`</ApiLink>: Extends `IRequestLoader` with write capabilities (adding and reclaiming requests).
 - <ApiLink to="core/class/RequestManagerTandem">`RequestManagerTandem`</ApiLink>: Combines a read-only `IRequestLoader` with a writable `IRequestManager`.
+- <ApiLink to="core/class/ThrottlingRequestManager">`ThrottlingRequestManager`</ApiLink>: Wraps a writable `IRequestManager` and paces requests per domain.
 
 And the concrete request loader implementations:
 
@@ -79,6 +80,8 @@ class SitemapRequestLoader
 
 class RequestManagerTandem
 
+class ThrottlingRequestManager
+
 %% ========================
 %% Inheritance arrows
 %% ========================
@@ -88,6 +91,7 @@ IRequestLoader <|.. RequestList
 IRequestLoader <|.. SitemapRequestLoader
 IRequestManager <|.. RequestQueue
 IRequestManager <|.. RequestManagerTandem
+IRequestManager <|.. ThrottlingRequestManager
 ```
 
 :::info Crawler usage
@@ -130,6 +134,42 @@ The loader supports filtering URLs using glob patterns and regular expressions,
 
 The <ApiLink to="core/interface/IRequestManager">`IRequestManager`</ApiLink> interface extends `IRequestLoader` with **write** capabilities. In addition to reading requests, a request manager can add new requests and reclaim failed ones. This is essential for dynamic crawling, where new URLs emerge during the crawl, or when requests fail and need to be retried. The <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> is the primary built-in request manager — see the [Request storage](./request-storage) guide for details.
 
+## Per-domain throttling
+
+Some sites answer bursts of traffic with HTTP 429 (Too Many Requests) rather than an outright block. By default a 429 is treated as a blocked session: the session is retired and the request is retried straight away on a fresh one, which churns through proxies without actually slowing down.
+
+The <ApiLink to="core/class/ThrottlingRequestManager">`ThrottlingRequestManager`</ApiLink> handles it at the scheduling layer instead. Wrap your request manager in it and list the domains you want paced:
+
+```ts
+import { CheerioCrawler, RequestQueue, ThrottlingRequestManager } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    requestManager: new ThrottlingRequestManager({
+        inner: await RequestQueue.open(),
+        domains: ['api.example.com'],
+        // optional, these are the defaults
+        baseDelaySecs: 2,
+        maxDelaySecs: 60,
+        maxDomainStallSecs: 900,
+    }),
+    requestHandler: async ({ request }) => {
+        // ...
+    },
+});
+```
+
+Requests for a listed domain are routed into their own queue as they are added. When one of those domains answers with a 429, the crawler honours its `Retry-After` header — or backs off exponentially from `baseDelaySecs` up to `maxDelaySecs` if there is none — and holds that domain's requests back for the duration. Requests for every other domain keep flowing at full speed, the throttled request is retried later without counting against `maxRequestRetries`, and its session is left alone, because a rate limit says nothing about the session.
+
+Because a throttled request costs no retries, a domain that never stops rate-limiting would otherwise keep the crawl alive forever. If one goes `maxDomainStallSecs` without letting a single request through, the crawl shuts down with a `PersistentRateLimitError` — at that point the concurrency is too high for that domain, or it has blocked you outright, and waiting longer will not help. Its requests are left in their queue on purpose, so re-running the crawl without purging storages resumes them if the rate limit lifts. A crawler running with `keepAlive` is exempt, since staying up regardless is what it was asked to do.
+
+This is opt-in and exact: only the domains you list are throttled, and matching is case-insensitive with no wildcard support, so list each subdomain you care about.
+
+:::note robots.txt crawl-delay
+
+`ThrottlingRequestManager` is also what enforces `Crawl-delay` directives when `respectRobotsTxtFile` is enabled. Without it, or for a domain missing from `domains`, the directive is ignored and the crawler warns you about it.
+
+:::
+
 ## Request manager tandem
 
 The <ApiLink to="core/class/RequestManagerTandem">`RequestManagerTandem`</ApiLink> class combines the read-only capabilities of an `IRequestLoader` (like <ApiLink to="core/class/RequestList">`RequestList`</ApiLink>) with the read-write capabilities of an `IRequestManager` (like <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>). This is useful when you need to load initial requests from a static source (such as a file, sitemap, or database) and also dynamically add or retry requests during the crawl.
@@ -174,6 +214,6 @@ Similarly, you can combine a <ApiLink to="core/class/SitemapRequestLoader">`Site
 
 ## Conclusion
 
-This guide introduced the request loader abstractions: the read-only <ApiLink to="core/interface/IRequestLoader">`IRequestLoader`</ApiLink>, the writable <ApiLink to="core/interface/IRequestManager">`IRequestManager`</ApiLink>, and the <ApiLink to="core/class/RequestManagerTandem">`RequestManagerTandem`</ApiLink> that combines them, along with the <ApiLink to="core/class/RequestList">`RequestList`</ApiLink> and <ApiLink to="core/class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> implementations. You also saw how to pair a loader with a queue using the `toTandem()` helper to handle both static and dynamically discovered requests.
+This guide introduced the request loader abstractions: the read-only <ApiLink to="core/interface/IRequestLoader">`IRequestLoader`</ApiLink>, the writable <ApiLink to="core/interface/IRequestManager">`IRequestManager`</ApiLink>, and the <ApiLink to="core/class/RequestManagerTandem">`RequestManagerTandem`</ApiLink> that combines them, along with the <ApiLink to="core/class/RequestList">`RequestList`</ApiLink> and <ApiLink to="core/class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> implementations. You also saw how to pair a loader with a queue using the `toTandem()` helper to handle both static and dynamically discovered requests, and how <ApiLink to="core/class/ThrottlingRequestManager">`ThrottlingRequestManager`</ApiLink> paces requests to individual domains.
 
 If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping!
diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx
@@ -213,6 +213,14 @@ const crawler = new CheerioCrawler({
 
 For sites that respond with a `200` page that is actually a bot wall (Cloudflare challenges, Google's rate-limit page), set `retryOnBlocked: true` to have the crawler detect those by content and retry as well. For deeper anti-blocking measures see the [avoid blocking guide](./avoid-blocking).
 
+:::tip A 429 is a rate limit, not a block
+
+Retiring a session on HTTP 429 burns proxies without slowing anything down — the site is asking you to wait, not telling you the session is unwelcome. Wrap your request manager in a <ApiLink to="core/class/ThrottlingRequestManager">`ThrottlingRequestManager`</ApiLink> to back off per domain instead of rotating; see [per-domain throttling](./request-loaders#per-domain-throttling). Sessions are left untouched for the domains it covers.
+
+Those domains are handled as rate limits before `blockedStatusCodes` is consulted, so leave 429 in the list — removing it only changes what happens for domains the manager does not cover.
+
+:::
+
 ## Sharing a session pool between crawlers
 
 A `SessionPool` instance can be shared across multiple crawlers by passing the same object to each crawler's `sessionPool` option. This is useful in multi-stage scrapers — for example a fast `CheerioCrawler` that discovers links and a `PlaywrightCrawler` that renders detail pages — where you want both stages to reuse the same proven, non-blocked identities and their cookies instead of each warming up its own pool from scratch.

diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
@@ -123,6 +123,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
@@ -159,6 +160,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
     pause(timeoutSecs?: number): Promise<void>;
     readonly proxyConfiguration?: IProxyConfiguration;
     pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
+    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
     // (undocumented)
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected requestManager?: IRequestManager;

diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
@@ -41,6 +41,7 @@ import { StringPredicate } from 'ow';
 // Not exported by the entry point; reachable only as a referenced type.
 // @public (undocumented)
 interface BaseResponse {
+    headers?(): Record<string, string>;
     // (undocumented)
     status(): number;
 }
@@ -79,6 +80,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
@@ -1149,6 +1149,9 @@ interface NewUrlOptions {
 export class NonRetryableError extends Error {
 }
 
+// @public
+export function parseRetryAfterHeader(value?: string | null): number | null;
+
 // @public
 export function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;
 
@@ -1160,6 +1163,10 @@ export interface PersistenceOptions {
     enable?: boolean;
 }
 
+// @public
+export class PersistentRateLimitError extends CriticalError {
+}
+
 // @public
 export class ProxyConfiguration implements IProxyConfiguration {
     constructor(options?: ProxyConfigurationOptions);
@@ -1336,6 +1343,9 @@ export interface RequestListState {
     nextUniqueKey: string | null;
 }
 
+// @public
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
+
 // @public
 export class RequestManagerTandem implements IRequestManager {
     // (undocumented)
@@ -1531,6 +1541,11 @@ export enum RequestState {
     UNPROCESSED = 0
 }
 
+// @public
+export class RequestThrottledError extends RetryRequestError {
+    constructor(message?: string);
+}
+
 // @public
 export interface RequestTransform {
     // (undocumented)
@@ -2086,6 +2101,19 @@ export interface StorageWritePolicy {
     requestQueue: StorageWriteMode;
 }
 
+// @public
+export interface SupportsDomainThrottling {
+    // (undocumented)
+    assertNoStalledDomains(): Promise<void>;
+    // (undocumented)
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    // (undocumented)
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+}
+
+// @public
+export function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+
 // @public
 export interface SystemInfo {
     // (undocumented)
@@ -2110,6 +2138,50 @@ export interface TaskLoopPredicates {
     isTaskReadyFunction?: () => Promise<boolean>;
 }
 
+// @public
+export class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    // (undocumented)
+    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    // (undocumented)
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    assertNoStalledDomains(): Promise<void>;
+    // (undocumented)
+    drop(): Promise<void>;
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request_2<R> | null>;
+    // (undocumented)
+    getHandledCount(): Promise<number>;
+    // (undocumented)
+    getPendingCount(): Promise<number>;
+    // (undocumented)
+    getTotalCount(): Promise<number>;
+    get innerManager(): T;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    // (undocumented)
+    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
+    // (undocumented)
+    persistState(): Promise<void>;
+    purge(): Promise<void>;
+    // (undocumented)
+    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    // (undocumented)
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+}
+
+// @public
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    baseDelaySecs?: number;
+    domains: string[];
+    inner: T;
+    maxDelaySecs?: number;
+    maxDomainStallSecs?: number;
+    requestManagerOpener?: RequestManagerOpener<T>;
+}
+
 export { tryAbsoluteURL }
 
 // @public

diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
@@ -128,6 +128,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
@@ -72,6 +72,7 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
@@ -361,6 +361,7 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedCon
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
@@ -215,6 +215,7 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedCont
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
@@ -103,6 +103,7 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont
         extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
         requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+        requestManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
         requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
         errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;

diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
@@ -152,6 +152,7 @@ export class RobotsTxtFile {
         logger?: CrawleeLogger;
     }): Promise<RobotsTxtFile>;
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
+    getCrawlDelay(userAgent?: string): number | undefined;
     getSitemaps(): string[];
     isAllowed(url: string, userAgent?: string): boolean;
     parseSitemaps(): Promise<Sitemap>;