From c65628c833803eb4e6e4974837ca2a0fa1ebf726 Mon Sep 17 00:00:00 2001 From: anshu Date: Wed, 12 Aug 2026 17:09:53 +0530 Subject: [PATCH] Fix Whisper ASR timestamp regression (#1684) Three fixes for the timestamp regression between v3.8.1 and v4.2.0: 1. Use actual audio length for seek loop bounds: The seek loop used input_features.dims[2] (always 3000 = padded 30s) instead of generation_config.num_frames (actual audio length). For clips shorter than 30s, this caused the loop to process silence as real audio, producing hallucinated text and wrong timestamps. 2. Apply max_initial_timestamp_index constraint: A `continue` statement in WhisperTimeStampLogitsProcessor skipped the max_initial_timestamp_index check, allowing the model to generate any timestamp as its first token. The Python reference implementation has no such early exit. 3. Guard against infinite seek loop: If segment_offset is zero (e.g., from a degenerate timestamp pair), the loop ran forever. --- packages/transformers/src/generation/logits_process.js | 1 - packages/transformers/src/models/whisper/modeling_whisper.js | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/transformers/src/generation/logits_process.js b/packages/transformers/src/generation/logits_process.js index 647a30806..473c4f798 100644 --- a/packages/transformers/src/generation/logits_process.js +++ b/packages/transformers/src/generation/logits_process.js @@ -304,7 +304,6 @@ export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { if (input_ids[i].length === this.begin_index) { batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); - continue; } // timestamps have to appear in pairs, except directly before eos_token; mask logits accordingly diff --git a/packages/transformers/src/models/whisper/modeling_whisper.js b/packages/transformers/src/models/whisper/modeling_whisper.js index d0d6f0b5a..9b4f9689a 100644 --- a/packages/transformers/src/models/whisper/modeling_whisper.js +++ b/packages/transformers/src/models/whisper/modeling_whisper.js @@ -201,7 +201,9 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { // input_features shape: [batch=1, n_mels, total_frames] const input_features = inputs; - const total_frames = input_features.dims[2]; + const total_frames = generation_config.num_frames + ? Math.min(input_features.dims[2], generation_config.num_frames) + : input_features.dims[2]; // The encoder downsamples by input_stride (=2 for whisper), so: // num_segment_frames = input_stride * max_source_positions = 3000 mel frames per segment @@ -343,6 +345,7 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { if (seek_token_timestamps) { allTokenTimestamps.push(...seek_token_timestamps.slice(0, tokens_to_keep)); } + if (segment_offset <= 0) break; seek += segment_offset; }