Load external data without copying it onto the JS heap

packages/transformers/package.json

@@ -58,7 +58,7 @@
     "@huggingface/jinja": "^0.5.6",
     "@huggingface/tokenizers": "^0.1.3",
     "onnxruntime-node": "1.24.3",
-    "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c",
+    "onnxruntime-web": "1.29.0-dev.20260811-e415ef9afd",
     "sharp": "^0.34.5"
   },
   "devDependencies": {

packages/transformers/src/utils/hub.js

@@ -245,9 +245,11 @@ export async function storeCachedResource(path_or_repo_id, filename, cache, cach
  * @param {PretrainedOptions} [options] An object containing optional parameters.
  * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content.
  * @param {import('./cache.js').CacheInterface | null} [cache] The cache instance to use.
+ * @param {boolean} [as_blob=false] Whether to return a `Blob` when the file is served from the cache,
+ * rather than reading it into a `Uint8Array`. Only honoured on a cache hit — see the note at the use site.
  *
  * @throws Will throw an error if the file is not found and `fatal` is true.
- * @returns {Promise<string|Uint8Array|null>} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true.
+ * @returns {Promise<string|Uint8Array|Blob|null>} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache.
  */
 export async function loadResourceFile(
     path_or_repo_id,
@@ -256,6 +258,7 @@ export async function loadResourceFile(
     options = {},
     return_path = false,
     cache = null,
+    as_blob = false,
 ) {
     const { requestURL, localPath, remoteURL, proposedCacheKey, validModelId } = buildResourcePaths(
         path_or_repo_id,
@@ -366,7 +369,79 @@ export async function loadResourceFile(
         let buffer;
 
         if (typeof response !== 'string') {
-            if (!options.progress_callback) {
+            if (as_blob && cacheHit) {
+                // The whole point of `as_blob`, and it is one call: a Cache Storage `Response` hands back a
+                // Blob that is a FILE REFERENCE rather than a copy, so the bytes never enter the JS heap.
+                //
+                // Gated on `cacheHit`, which is a deliberate trade rather than caution. Reading the stream to
+                // report progress would defeat the whole thing — the chunks would be resident twice, once as
+                // buffers and once in Blob storage — and on a cold download progress is worth more than peak
+                // memory, because the user is watching several gigabytes arrive. Every load AFTER the first
+                // takes this branch, is instantaneous, and is where the peak actually matters.
+                buffer = /** @type {any} */ (await response.blob());
+
+                dispatchCallback(options.progress_callback, {
+                    status: 'progress',
+                    name: path_or_repo_id,
+                    file: filename,
+                    progress: 100,
+                    loaded: buffer.size,
+                    total: buffer.size,
+                });
+            } else if (as_blob && toCacheResponse && response.body) {
+                // COLD, and headed for the cache anyway. Stream the body straight into Cache Storage and then
+                // read it back as a Blob, so the bytes go network -> disk -> runtime and never sit on the JS
+                // heap at all.
+                //
+                // This is the case that actually fails. `getModelDataFiles` starts every external-data chunk
+                // concurrently, and reading each one into a `Uint8Array` to report progress means a cold load
+                // peaks at the SUM of the chunks: a 17 GB model raises `Array buffer allocation failed` at
+                // ~16 GB and only completes on a later attempt, once enough files are cached to take the
+                // branch above.
+                //
+                // Progress survives, which is the reason the buffer existed. A pass-through `TransformStream`
+                // counts bytes as they go by; it holds one chunk, not the file, and backpressure keeps it
+                // that way.
+                let loaded = 0;
+                const total = parseInt(response.headers.get('content-length'), 10) || 0;
+                const counting = new TransformStream({
+                    transform(chunk, controller) {
+                        loaded += chunk.byteLength;
+                        dispatchCallback(options.progress_callback, {
+                            status: 'progress',
+                            name: path_or_repo_id,
+                            file: filename,
+                            progress: total ? (loaded / total) * 100 : 0,
+                            loaded,
+                            total,
+                        });
+                        controller.enqueue(chunk);
+                    },
+                });
+
+                // `content-length` explicitly, because the Cache API may strip it — same reason the buffered
+                // store below sets it.
+                const headers = new Headers(response.headers);
+                if (total) headers.set('content-length', String(total));
+
+                try {
+                    await cache.put(cacheKey, new Response(response.body.pipeThrough(counting), { headers }));
+                    const stored = await cache.match(cacheKey);
+                    if (!stored) throw new Error('cache.match missed the entry just written');
+                    buffer = /** @type {any} */ (await stored.blob());
+                    // Already stored, so the block at the end of this function must not store it again.
+                    toCacheResponse = false;
+                } catch (err) {
+                    // The buffered path keeps working when the cache refuses the write (QuotaExceededError is
+                    // the expected one). It cannot reuse `response` — the body is consumed — so it re-fetches.
+                    logger.warn(`Unable to stream response into the cache, falling back to a buffer: ${err}.`);
+                    // `getFile`, not bare `fetch`: it routes through `env.fetch` and applies
+                    // `getFetchHeaders`, so a gated repo keeps its Authorization header on the way back.
+                    const retry = await getFile(remoteURL);
+                    if (retry.status !== 200) return handleError(retry.status, remoteURL, fatal);
+                    buffer = new Uint8Array(await retry.arrayBuffer());
+                }
+            } else if (!options.progress_callback) {
                 // If no progress callback is specified, we can use the `.arrayBuffer()`
                 // method to read the response.
                 buffer = new Uint8Array(await response.arrayBuffer());
@@ -494,11 +569,20 @@ const INFLIGHT_LOADS = new Map();
  * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.
  * @param {PretrainedOptions} [options] An object containing optional parameters.
  * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content.
+ * @param {boolean} [as_blob=false] Whether to accept a `Blob` for a file served from the cache, avoiding a
+ * copy of its bytes on the JS heap. Intended for external data, which is handed straight to the runtime.
  *
  * @throws Will throw an error if the file is not found and `fatal` is true.
- * @returns {Promise<string|Uint8Array>} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true.
+ * @returns {Promise<string|Uint8Array|Blob>} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache.
  */
-export async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}, return_path = false) {
+export async function getModelFile(
+    path_or_repo_id,
+    filename,
+    fatal = true,
+    options = {},
+    return_path = false,
+    as_blob = false,
+) {
     if (!env.allowLocalModels) {
         // User has disabled local models, so we just make sure other settings are correct.
 
@@ -529,7 +613,7 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti
             file: filename,
         });
         pending = getCache(options.cache_dir).then((cache) =>
-            loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache),
+            loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache, as_blob),
         );
         if (loads === INFLIGHT_LOADS) {
             pending = pending.finally(() => INFLIGHT_LOADS.delete(key));

packages/transformers/src/utils/model-loader.js

@@ -57,7 +57,7 @@ export async function getCoreModelFile(pretrained_model_name_or_path, fileName,
  * @param {import('./hub.js').PretrainedModelOptions} options Additional options for loading the model.
  * @param {import('./hub.js').ExternalData|Record<string, import('./hub.js').ExternalData>|undefined} use_external_data_format External data format configuration.
  * @param {any} [session_options] Optional session options that may contain externalData configuration.
- * @returns {Promise<Array<string|{path: string, data: Uint8Array}>>} A Promise that resolves to an array of external data files.
+ * @returns {Promise<Array<string|{path: string, data: Uint8Array|Blob}>>} A Promise that resolves to an array of external data files.
  */
 export async function getModelDataFiles(
     pretrained_model_name_or_path,
@@ -70,7 +70,7 @@ export async function getModelDataFiles(
     const baseName = `${fileName}${suffix}.onnx`;
     const return_path = apis.IS_NODE_ENV;
 
-    /** @type {Promise<string|{path: string, data: Uint8Array}>[]} */
+    /** @type {Promise<string|{path: string, data: Uint8Array|Blob}>[]} */
     let externalDataPromises = [];
 
     const num_chunks = resolveExternalDataFormat(use_external_data_format, baseName, fileName);
@@ -91,8 +91,11 @@ export async function getModelDataFiles(
                         true,
                         options,
                         return_path,
+                        // In the browser, hand onnxruntime-web a Blob rather than a materialised buffer.
+                        // Node keeps returning a path, which is cheaper still.
+                        !return_path,
                     );
-                    resolve(data instanceof Uint8Array ? { path, data } : path);
+                    resolve(data instanceof Uint8Array || data instanceof Blob ? { path, data } : path);
                 }),
             );
         }