From 6a6aee46d2ba9b77fe3661fc4ee41f3b13530896 Mon Sep 17 00:00:00 2001 From: Parth Maheshwari Date: Tue, 11 Aug 2026 15:53:53 +0100 Subject: [PATCH 1/4] [NPUW] Remove NPUW-side Whisper cross-attention SDPA decomposition DO NOT MERGE before the corresponding openvino.genai change that makes GenAI decompose cross-attention SDPA for NPU too (matching CPU/GPU) is merged - this removes NPUW's own decomposition, so until then Whisper word-level timestamps on NPU would break. With the previous commit making the Whisper transformation passes tolerant of already-decomposed cross-attention SDPA, NPUW no longer needs to decompose it internally after those passes have run: - Removed WhisperScaledDotProductAttentionDecomposition and its helper can_move_scale_after_matmul, decompose_scaled_dot_product_attention_ for_whisper(), and add_cross_attention_qk_scaled_scores_outputs_for_ whisper() from prepare_whisper_model.cpp. - Removed the NPUW_WHISPER_DECOMPOSE_SDPA option, PrepareWhisperPrefill Model's decompose_sdpa parameter, and the whisper_kvcache_size - 1 adjustment that only applied in that path. - Removed the fake dangling Parameter->Result pairs llm_compiled_model.cpp used to mock "cross_attention_qk_scaled_scores_N" outputs on the compiled model - once GenAI adds these as real outputs before NPUW compiles the model, no mocking is needed. - WhisperInferRequest now discovers those outputs by tensor name from m_prefill_out_ports instead of a precomputed decomposed-layer count, since NPUW itself no longer produces them. - Bumped NPUW_SERIALIZATION_VERSION (blob format changed: dropped m_decomposed_sdpa_size). CVS-184242 --- .../intel_npu/config/npuw_option_defs.inc | 1 - .../src/plugin/npuw/llm_compiled_model.cpp | 34 +-- .../src/plugin/npuw/llm_compiled_model.hpp | 1 - .../src/plugin/npuw/serialization.hpp | 2 +- .../npuw/whisper/prepare_whisper_model.cpp | 249 +----------------- .../npuw/whisper/prepare_whisper_model.hpp | 13 +- .../npuw/whisper/whisper_infer_request.cpp | 33 ++- .../npuw/whisper/whisper_infer_request.hpp | 6 +- ...lm_compiled_model_factory_options_test.cpp | 3 +- .../unit/npuw/prepare_whisper_model_test.cpp | 4 +- 10 files changed, 40 insertions(+), 306 deletions(-) diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc b/src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc index 081be721abc4b8..28d691465fc480 100644 --- a/src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc +++ b/src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc @@ -84,7 +84,6 @@ INTEL_NPU_NPUW_ANYMAP_OPT(NPUW_LLM_SHARED_LM_HEAD_CONFIG, ov::intel_npu::npuw::l INTEL_NPU_NPUW_ANYMAP_OPT(NPUW_LLM_ADDITIONAL_SHARED_LM_HEAD_CONFIG, ov::intel_npu::npuw::llm, additional_shared_lm_head_config, "++NPUW_LLM_SHARED_HEAD_CONFIG", LLM, EXPOSED, CACHED, ALL) INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_WHISPER, bool, false, ov::intel_npu::npuw::whisper, enabled, "NPUW_WHISPER", LLM, EXPOSED, CACHED, ALL) INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_WHISPER_EOS_TOKEN, uint64_t, 50257, ov::intel_npu::npuw::whisper, whisper_eos_token, "NPUW_WHISPER_EOS_TOKEN", LLM, EXPOSED, UNCACHED, ALL) -INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_WHISPER_DECOMPOSE_SDPA, bool, false, ov::intel_npu::npuw::whisper, whisper_decompose_sdpa, "NPUW_WHISPER_DECOMPOSE_SDPA", LLM, EXPOSED, CACHED, ALL) INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_EAGLE, bool, false, ov::intel_npu::npuw::eagle, enabled, "NPUW_EAGLE", LLM, EXPOSED, CACHED, ALL) INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_TEXT_EMBED, bool, false, ov::intel_npu::npuw::text_embed, enabled, "NPUW_TEXT_EMBED", LLM, EXPOSED, CACHED, ALL) INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_KOKORO, bool, false, ov::intel_npu::npuw::kokoro, enabled, "NPUW_KOKORO", KOKORO, EXPOSED, UNCACHED, ALL) diff --git a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp index 7a3f938b18a596..6ca2173e9586ed 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp @@ -853,32 +853,10 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr& m m_kvcache_desc = KVCacheDesc{whisper_max_prompt_size, whisper_kvcache_size, 0u, whisper_seq_len_dim, 1u}; whisper_lhs_seq_size = static_cast(prefill_model->input("encoder_hidden_states").get_partial_shape()[1].get_length()); - auto whisper_decompose_sdpa = m_cfg.get<::intel_npu::NPUW_WHISPER_DECOMPOSE_SDPA>(); - if (whisper_decompose_sdpa) { - m_kvcache_desc.max_prompt_size = whisper_kvcache_size - 1; - } - auto prepare_prefill_model = ov::npuw::util::PrepareWhisperPrefillModel(m_kvcache_desc.max_prompt_size, - whisper_lhs_seq_size, - whisper_decompose_sdpa); - prepare_prefill_model.run_on_model(prefill_model); // Whisper decoder model + ov::npuw::util::PrepareWhisperPrefillModel(m_kvcache_desc.max_prompt_size, whisper_lhs_seq_size) + .run_on_model(prefill_model); // Whisper decoder model ov::npuw::util::PrepareWhisperKVCacheModel().run_on_model(kvcache_model); // Whisper decoder_with_past model - - // FIXME: Whisper Decompose SDPA - // WA: to mock new "cross_attention_qk_scaled_scores" outputs in original model - if (whisper_decompose_sdpa) { - m_decomposed_sdpa_size = prepare_prefill_model.get_decomposed_sdpa_size(); - auto& mutable_outputs = const_cast>&>(this->outputs()); - for (size_t idx = 0; idx < m_decomposed_sdpa_size; idx++) { - auto fake_param = std::make_shared(ov::element::f32, ov::PartialShape{}); - auto fake_result = std::make_shared(fake_param); - fake_result->output(0).get_tensor().add_names( - {WhisperInferRequest::whisper_layer_names::qk_scores, - WhisperInferRequest::whisper_layer_names::qk_scores_ + std::to_string(idx)}); - - mutable_outputs.emplace_back(fake_result->output(0)); - } - } } LOG_DEBUG("Make prefill model with static shapes"); @@ -1316,8 +1294,8 @@ void ov::npuw::LLMCompiledModel::serialize(std::ostream& raw_stream, const ov::n m_kvcache_desc.dim & m_kvcache_desc.max_generation_token_len & m_kvcache_desc.v_tensors_transposed_pre & m_kvcache_desc.v_tensors_transposed_gen & m_prefill_chunk_size & m_use_chunk_prefill & m_max_lora_rank & m_enable_prefix_caching & m_prefix_caching_block_size & m_prefix_caching_max_num_blocks & - m_longrope_context_limit & m_is_whisper & m_eos_token_id & m_decomposed_sdpa_size & m_is_eagle & - m_is_embedding & m_is_block_kv_cache; + m_longrope_context_limit & m_is_whisper & m_eos_token_id & m_is_eagle & m_is_embedding & + m_is_block_kv_cache; // Write config stream & m_cfg; @@ -1537,8 +1515,7 @@ std::shared_ptr ov::npuw::LLMCompiledModel::deserial compiled->m_use_chunk_prefill & compiled->m_max_lora_rank & compiled->m_enable_prefix_caching & compiled->m_prefix_caching_block_size & compiled->m_prefix_caching_max_num_blocks & compiled->m_longrope_context_limit & compiled->m_is_whisper & compiled->m_eos_token_id & - compiled->m_decomposed_sdpa_size & compiled->m_is_eagle & compiled->m_is_embedding & - compiled->m_is_block_kv_cache; + compiled->m_is_eagle & compiled->m_is_embedding & compiled->m_is_block_kv_cache; // Deserialize config stream & compiled->m_cfg; @@ -1672,7 +1649,6 @@ void ov::npuw::LLMCompiledModel::implement_properties() { BIND(npuw::llm::shared_lm_head, NPUW_LLM_SHARED_HEAD, get), BIND(npuw::whisper::enabled, NPUW_WHISPER, get), BIND(npuw::whisper::whisper_eos_token, NPUW_WHISPER_EOS_TOKEN, get), - BIND(npuw::whisper::whisper_decompose_sdpa, NPUW_WHISPER_DECOMPOSE_SDPA, get), BIND(npuw::eagle::enabled, NPUW_EAGLE, get), BIND(npuw::text_embed::enabled, NPUW_TEXT_EMBED, get)}); #undef BIND diff --git a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp index d9796d2d444ae2..036f7c33928bc4 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.hpp @@ -139,7 +139,6 @@ class LLMCompiledModel : public ov::npuw::ICompiledModel { bool m_is_whisper = false; uint64_t m_eos_token_id = 0; - size_t m_decomposed_sdpa_size = 0; bool m_is_embedding = false; diff --git a/src/plugins/intel_npu/src/plugin/npuw/serialization.hpp b/src/plugins/intel_npu/src/plugin/npuw/serialization.hpp index c4f613a7ed7895..91aa6c9da8058e 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/serialization.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/serialization.hpp @@ -48,7 +48,7 @@ const constexpr ov::npuw::s11n::IndicatorType NPUW_LLM_COMPILED_MODEL_INDICATOR const constexpr ov::npuw::s11n::IndicatorType NPUW_GQA_COMPILED_MODEL_INDICATOR = {char{0x47}, char{0x51}, char{0x41}, char{0x43}, char{0x4d}, char{0x4f}}; -const constexpr char* NPUW_SERIALIZATION_VERSION = "0.28"; +const constexpr char* NPUW_SERIALIZATION_VERSION = "0.29"; // Forward declaration namespace intel_npu { diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp index 35706ab086cbcb..3166017a8631ae 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp @@ -13,7 +13,6 @@ #include "openvino/openvino.hpp" #include "openvino/opsets/opset13.hpp" #include "openvino/pass/graph_rewrite.hpp" -#include "openvino/pass/manager.hpp" #include "openvino/pass/matcher_pass.hpp" #include "openvino/pass/pattern/op/optional.hpp" #include "openvino/pass/pattern/op/or.hpp" @@ -102,10 +101,10 @@ class AttentionMaskInputPast_2 : public ov::pass::MatcherPass { } }; -// If cross-attention SDPA has already been decomposed (by GenAI, or by -// decompose_scaled_dot_product_attention_for_whisper() below), there's no SDPA node left -// to find for it. The decomposition always tags the "QK scaled scores" node with this -// well-known tensor name, so it doubles as a reliable marker for where the block lives. +// If GenAI has already decomposed cross-attention SDPA (for word-level timestamps), +// there's no SDPA node left to find for it. The decomposition always tags the "QK +// scaled scores" node with this well-known tensor name, so it doubles as a reliable +// marker for where the block lives. std::vector> find_decomposed_cross_attn_score_nodes( const std::shared_ptr& model) { std::vector> found; @@ -338,198 +337,10 @@ class CachePositionInput_2 : public ov::pass::MatcherPass { } }; -bool can_move_scale_after_matmul(const ov::Output& query, - const ov::Output& kT, - const ov::Output& scale) { - const auto& scale_pshape = scale.get_partial_shape(); - const auto& query_pshape = query.get_partial_shape(); - if (scale_pshape.is_dynamic() || query_pshape.is_dynamic()) { - return false; - } - - // According to the ov SDPA specification, the scale input have to be 1d with 1 element - // or scalar. - if (ov::shape_size(scale_pshape.to_shape()) != 1) { - return false; - } - - // using the original implementation to calculate the shapes. - // we need to move the scale after MatMul only if the tensor after MatMul is smaller. - auto q_scaled = std::make_shared(query, scale); - auto scaled_attn = std::make_shared(q_scaled, kT); - const auto& scaled_attn_pshape = scaled_attn->output(0).get_partial_shape(); - if (scaled_attn_pshape.is_static()) { - return ov::shape_size(query_pshape.to_shape()) > ov::shape_size(scaled_attn_pshape.to_shape()); - } - return false; -} - -// FIXME: Whisper Decompose SDPA -class WhisperScaledDotProductAttentionDecomposition : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("WhisperScaledDotProductAttentionDecomposition"); - WhisperScaledDotProductAttentionDecomposition() { - auto pattern_node = ov::pass::pattern::wrap_type(); - - ov::matcher_pass_callback callback = [this, pattern_node](ov::pass::pattern::Matcher& m) { - auto& pattern_to_output = m.get_pattern_value_map(); - - auto node = ov::as_type_ptr( - pattern_to_output.at(pattern_node).get_node_shared_ptr()); - - if (node == nullptr || transformation_callback(node)) { - return false; - } - - const std::string& node_name = node->get_friendly_name(); - if (node_name.find("encoder_attn") == std::string::npos) { - // This pass is only for encoder-decoder cross-attention layers - return false; - } - - auto new_output_node = decompose(node); - ov::replace_node(node, new_output_node); - return true; - }; - - auto m = std::make_shared(pattern_node, - "WhisperScaledDotProductAttentionDecompositionMatcher"); - register_matcher(m, callback); - } - - std::shared_ptr decompose(std::shared_ptr node) { - using namespace ov::op; - using namespace ov; - auto query = node->input_value(0); - auto key = node->input_value(1); - auto value = node->input_value(2); - auto q_shape = register_new_node(query, element::i32); - auto k_shape = register_new_node(key, element::i32); - auto minus_one = register_new_node(v0::Constant::create(element::i32, Shape{}, {-1})); - auto minus_two = register_new_node(v0::Constant::create(element::i32, Shape{}, {-2})); - auto zero_i = register_new_node(v0::Constant::create(element::i32, Shape{}, {0})); - auto one_i = register_new_node(v0::Constant::create(element::i32, Shape{}, {1})); - auto one_f = register_new_node(one_i, query); - auto zero_f = register_new_node(zero_i, query); - - auto build_extract_dim_subgraph = [this, &zero_i](const std::shared_ptr& shape_of, - const int64_t idx) -> std::shared_ptr { - const auto dim_to_extract_const = v0::Constant::create(element::i32, Shape{}, {idx}); - const auto gather = std::make_shared(shape_of, dim_to_extract_const, zero_i); - - register_new_node(dim_to_extract_const); - return register_new_node(gather); - }; - - Output scale; - Output sink; - bool has_sink = false; - if (node->get_input_size() < 5) { - scale = build_extract_dim_subgraph(q_shape, -1); - scale = register_new_node(scale, query); - auto sqrt_scale = register_new_node(scale); - scale = register_new_node(one_f, sqrt_scale); - } else { - scale = node->input_value(4); - if (node->get_input_size() == 6) { - sink = node->input_value(5); - has_sink = true; - } - } - - auto k_rank = register_new_node(k_shape, element::i32)->output(0); - auto k_last_dim = register_new_node(k_rank, minus_one); - auto k_next_dim = register_new_node(k_rank, minus_two)->output(0); - k_rank = register_new_node(k_rank, zero_i); - auto minus_inf = - register_new_node(v0::Constant::create(element::f32, Shape{}, {-std::numeric_limits::infinity()})) - ->output(0); - auto keep_dim_last = register_new_node(k_next_dim, zero_i); - auto k_dims_before_transpose = register_new_node(zero_i, keep_dim_last, one_i, element::i32); - - auto transpose_dims = - register_new_node(OutputVector{k_dims_before_transpose, k_last_dim, k_next_dim}, 0); - auto k_transposed = register_new_node(key, transpose_dims); - - ov::Output scaled_atten; - if (can_move_scale_after_matmul(query, k_transposed, scale)) { - auto atten = register_new_node(query, k_transposed)->output(0); - scaled_atten = register_new_node(atten, scale)->output(0); - } else { - auto q_scaled = register_new_node(query, scale); - scaled_atten = register_new_node(q_scaled, k_transposed)->output(0); - } - - minus_inf = register_new_node(minus_inf, scaled_atten); - - if (node->get_causal() || node->get_input_size() > 3) { - Output mask; - Output atten_mask; - if (!node->get_causal()) { - mask = node->input_value(3); - - // two types of masks are supported. A boolean mask where a value of True indicates that the element - // should take part in attention. A float mask of the same type as query, key, value that is added to - // the attention score. - if (mask.get_element_type() == element::boolean) { - atten_mask = register_new_node(mask, zero_f, minus_inf); - } else { - atten_mask = mask; - } - } else { - auto target_s_len = build_extract_dim_subgraph(q_shape, -2); - auto source_s_len = build_extract_dim_subgraph(k_shape, -2); - auto ssl = register_new_node(source_s_len, zero_i); - auto tsl = register_new_node(target_s_len, zero_i); - auto mask_shape = register_new_node(OutputVector{tsl, ssl}, 0); - mask = register_new_node(minus_inf, mask_shape); - auto horizontal_range = - register_new_node(zero_i, source_s_len, one_i, element::i32)->output(0); - horizontal_range = register_new_node(horizontal_range, zero_i); - auto stop = register_new_node(target_s_len, one_i); - auto vertical_range = register_new_node(one_i, stop, one_i, element::i32)->output(0); - vertical_range = register_new_node(vertical_range, one_i); - auto triu = register_new_node(horizontal_range, vertical_range); - atten_mask = register_new_node(triu, mask, zero_f); - } - scaled_atten = register_new_node(scaled_atten, atten_mask); - } - - scaled_atten.add_names({"cross_attention_qk_scaled_scores"}); - - if (has_sink) { - auto minus_two = register_new_node(v0::Constant::create(element::i32, Shape{1}, {-2})); - auto minus_one = register_new_node(v0::Constant::create(element::i32, Shape{1}, {-1})); - auto zero_i = register_new_node(v0::Constant::create(element::i32, Shape{1}, {0})); - auto one_i = register_new_node(v0::Constant::create(element::i32, Shape{1}, {1})); - - auto q_last_but_one_dim = register_new_node(register_new_node(q_shape), - v0::Constant::create(element::i64, Shape{}, {1})); - auto sink_target_shape_1 = register_new_node(q_shape, zero_i, q_last_but_one_dim, one_i); - auto sink_target_shape = register_new_node(OutputVector{sink_target_shape_1, one_i}, 0); - auto sink_broadcast = register_new_node(sink, sink_target_shape); - - auto scaled_attn_sink = register_new_node(OutputVector{scaled_atten, sink_broadcast}, -1); - scaled_atten = register_new_node(scaled_attn_sink, -1); - - auto prev_seq_len = register_new_node(k_shape, minus_two, zero_i); - scaled_atten = register_new_node(scaled_atten, zero_i, prev_seq_len, one_i, minus_one); - } else { - scaled_atten = register_new_node(scaled_atten, -1); - } - - auto result = register_new_node(scaled_atten, value); - result->set_friendly_name(node->get_friendly_name()); - copy_runtime_info(node, get_new_nodes()); - return result; - } -}; - // A cross-attention KV-cache state (the ReadValue producing the once-computed // encoder key/value) is consumed differently depending on whether the model -// still has a fused cross-attention SDPA node, or whether cross-attention SDPA -// has already been decomposed (by GenAI, or by -// decompose_scaled_dot_product_attention_for_whisper() below): +// still has a fused cross-attention SDPA node, or whether GenAI has already +// decomposed cross-attention SDPA (for word-level timestamps): // - fused: ReadValue -> [FakeConvert ->] SDPA (key at port 1, value at port 2) // - decomposed: key feeds the Transpose that builds "kT" for QK^T (and, // separately, a ShapeOf reading the same state to size that @@ -790,48 +601,6 @@ void add_cache_position_input(const std::shared_ptr& model) { ov::pass::Validate().run_on_model(model); } -// FIXME: Whisper Decompose SDPA -void decompose_scaled_dot_product_attention_for_whisper(std::shared_ptr model) { - ov::pass::Manager manager; - manager.register_pass(); - manager.run_passes(model); -} - -// FIXME: Whisper Decompose SDPA -size_t add_cross_attention_qk_scaled_scores_outputs_for_whisper(std::shared_ptr model) { - size_t idx = 0; - for (auto& op : model->get_ordered_ops()) { - if (op->get_type_info().name != std::string("Add")) { - continue; - } - - bool should_skip_op = true; - - for (const auto& output : op->outputs()) { - for (const auto& name : output.get_names()) { - if (name.find("cross_attention_qk_scaled_scores") != std::string::npos) { - should_skip_op = false; - break; - } - } - - // output found, exit outputs loop - if (!should_skip_op) { - break; - } - } - - if (should_skip_op) { - continue; - } - - model->add_output(op->output(0)).set_names({"cross_attention_qk_scaled_scores_" + std::to_string(idx)}); - idx++; - } - - return idx; -} - #ifdef __GNUC__ # pragma GCC diagnostic pop #endif @@ -852,12 +621,6 @@ bool ov::npuw::util::PrepareWhisperPrefillModel::run_on_model(const std::shared_ add_attention_mask_input(model, m_max_prompt_size, m_lhs_seq_size, true); - // FIXME: Whisper Decompose SDPA - if (m_decompose_sdpa) { - decompose_scaled_dot_product_attention_for_whisper(model); - m_decomposed_layers_size = add_cross_attention_qk_scaled_scores_outputs_for_whisper(model); - } - model->validate_nodes_and_infer_types(); return true; diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp index 9a9bafb765a757..69265be42d4923 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp @@ -16,23 +16,14 @@ namespace util { class PrepareWhisperPrefillModel : public ov::pass::ModelPass { uint32_t m_max_prompt_size; uint32_t m_lhs_seq_size; - bool m_decompose_sdpa; - size_t m_decomposed_layers_size; public: OPENVINO_MODEL_PASS_RTTI("ov::npuw::PrepareWhisperPrefillModel"); - explicit PrepareWhisperPrefillModel(uint32_t max_prompt_size, uint32_t lhs_seq_size, bool decompose_sdpa) + explicit PrepareWhisperPrefillModel(uint32_t max_prompt_size, uint32_t lhs_seq_size) : m_max_prompt_size(max_prompt_size), - m_lhs_seq_size(lhs_seq_size), - m_decompose_sdpa(decompose_sdpa), - m_decomposed_layers_size(0) {} + m_lhs_seq_size(lhs_seq_size) {} bool run_on_model(const std::shared_ptr& model) override; - - // FIXME: Whisper Decompose SDPA - size_t get_decomposed_sdpa_size() const { - return m_decomposed_layers_size; - } }; class PrepareWhisperKVCacheModel : public ov::pass::ModelPass { diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp index 0044d1cd48836c..8218069bb31b35 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp @@ -58,11 +58,16 @@ void ov::npuw::WhisperInferRequest::infer_prefill(ov::SoPtr input_i 0u, m_npuw_llm_compiled_model->m_kvcache_desc.num_stored_tokens); - // for word-level timestamps - auto decomposed_sdpa_size = m_npuw_llm_compiled_model->m_decomposed_sdpa_size; - for (size_t idx = 0; idx < decomposed_sdpa_size; idx++) { - auto name = whisper_layer_names::qk_scores_ + std::to_string(idx); - m_alignment_tensors.insert({name, m_prefill_request->get_tensor(m_prefill_out_ports.at(name))}); + // For word-level timestamps: the prefill submodel exposes one + // "cross_attention_qk_scaled_scores_N" output per decoder layer (GenAI decomposes + // cross-attention SDPA and adds these outputs before NPUW ever sees the model), found + // by name rather than by a precomputed count since NPUW no longer decomposes anything + // itself. Cache them here since get_tensor() is called per-port and cross-attention + // (unlike self-attention) only runs during prefill. + for (const auto& [name, port] : m_prefill_out_ports) { + if (port.get_names().count(WhisperInferRequest::whisper_layer_names::qk_scores) > 0) { + m_alignment_tensors.insert({name, m_prefill_request->get_tensor(port)}); + } } LOG_DEBUG("Done"); @@ -220,20 +225,20 @@ void ov::npuw::WhisperInferRequest::infer() { } } -// FIXME: Whisper Decompose SDPA ov::SoPtr ov::npuw::WhisperInferRequest::get_tensor(const ov::Output& port) const { const auto& port_names = port.get_names(); if (port_names.count(whisper_layer_names::qk_scores) > 0) { - for (auto name : port_names) { - if (name.find(whisper_layer_names::qk_scores_) != std::string::npos) { - auto alignment_tensor = m_alignment_tensors.at(name); - if (!alignment_tensor) { - OPENVINO_THROW( - "Cross-attention qk scaled scores tensor is not available. Please run inference first."); - } - return alignment_tensor; + for (const auto& name : port_names) { + auto it = m_alignment_tensors.find(name); + if (it == m_alignment_tensors.end()) { + continue; + } + if (!it->second) { + OPENVINO_THROW( + "Cross-attention qk scaled scores tensor is not available. Please run inference first."); } + return it->second; } } diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.hpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.hpp index 9479aaa2802b82..24197cce8a88e6 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.hpp @@ -12,8 +12,9 @@ namespace npuw { class WhisperInferRequest final : public LLMInferRequest { public: struct whisper_layer_names { + // Tensor name GenAI's cross-attention SDPA decomposition attaches to each decoder + // layer's "QK scaled scores" output, alongside a layer-specific "..._N" name. static constexpr const char* qk_scores = "cross_attention_qk_scaled_scores"; - static constexpr const char* qk_scores_ = "cross_attention_qk_scaled_scores_"; }; explicit WhisperInferRequest(const std::shared_ptr& compiled_model) @@ -31,6 +32,9 @@ class WhisperInferRequest final : public LLMInferRequest { void infer_generate(ov::SoPtr input_ids); bool m_need_copy_kvcache = false; + // For word-level timestamps: keyed by whatever name m_prefill_out_ports stores the + // port under, populated in infer_prefill() - see there for why this can't just be a + // generic get_tensor() passthrough. std::map> m_alignment_tensors{}; }; diff --git a/src/plugins/intel_npu/tests/unit/npuw/llm_compiled_model_factory_options_test.cpp b/src/plugins/intel_npu/tests/unit/npuw/llm_compiled_model_factory_options_test.cpp index 2ea20aeb9bcd9e..1d197fe2d1eaea 100644 --- a/src/plugins/intel_npu/tests/unit/npuw/llm_compiled_model_factory_options_test.cpp +++ b/src/plugins/intel_npu/tests/unit/npuw/llm_compiled_model_factory_options_test.cpp @@ -834,8 +834,7 @@ TEST_F(LLMCompiledModelFactoryOptionsTest, WhisperPrefillPreparationAddsCrossAtt EXPECT_TRUE(ov::npuw::util::PrepareWhisperPrefillModel( 128, - static_cast(ov::test::npuw::WhisperConfig{}.max_source_positions), - false /*decompose_sdpa*/) + static_cast(ov::test::npuw::WhisperConfig{}.max_source_positions)) .run_on_model(model)); auto prepared = model; diff --git a/src/plugins/intel_npu/tests/unit/npuw/prepare_whisper_model_test.cpp b/src/plugins/intel_npu/tests/unit/npuw/prepare_whisper_model_test.cpp index ef6e6799f15189..f4262eaaf1c8ce 100644 --- a/src/plugins/intel_npu/tests/unit/npuw/prepare_whisper_model_test.cpp +++ b/src/plugins/intel_npu/tests/unit/npuw/prepare_whisper_model_test.cpp @@ -110,9 +110,7 @@ TEST_P(PrepareWhisperModelTest, PrefillPreparationHandlesFusedAndDecomposedCross } WhisperConfig cfg; - ASSERT_TRUE(ov::npuw::util::PrepareWhisperPrefillModel(128, - static_cast(cfg.max_source_positions), - false /*decompose_sdpa*/) + ASSERT_TRUE(ov::npuw::util::PrepareWhisperPrefillModel(128, static_cast(cfg.max_source_positions)) .run_on_model(model)); EXPECT_TRUE(has_input_name(model, "attention_mask")); From ea2553d782e72299e90699f7228e5ecf7dc15544 Mon Sep 17 00:00:00 2001 From: Parth Maheshwari Date: Thu, 13 Aug 2026 13:57:46 +0100 Subject: [PATCH 2/4] [NPUW] Address review: restore whisper_kvcache_size-1 adjustment, trim comment Addresses eshiryae's review comments on the cross-attention SDPA decomposition cleanup: - Restored the whisper_kvcache_size - 1 static-prompt-length adjustment, which is needed whenever word-level timestamps are active (the decoder may receive tokens decoded from the entire audio chunk in one prefill call), not just as an artifact of NPUW's removed internal decompose option. Since that option is gone, detect the condition structurally via a new has_decomposed_cross_attention_sdpa() helper instead. - Trimmed an overly detailed comment in whisper_infer_request.cpp. --- .../intel_npu/src/plugin/npuw/llm_compiled_model.cpp | 7 +++++++ .../src/plugin/npuw/whisper/prepare_whisper_model.cpp | 4 ++++ .../src/plugin/npuw/whisper/prepare_whisper_model.hpp | 6 ++++++ .../src/plugin/npuw/whisper/whisper_infer_request.cpp | 7 +------ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp index 6ca2173e9586ed..f9fbba953842d6 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/llm_compiled_model.cpp @@ -854,6 +854,13 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr& m whisper_lhs_seq_size = static_cast(prefill_model->input("encoder_hidden_states").get_partial_shape()[1].get_length()); + // With word-level timestamps, the decoder may receive tokens decoded from the + // entire audio chunk in one prefill call, so the static prompt length needs to + // reach almost the full KV-cache size rather than the usual (smaller) max prompt. + if (ov::npuw::util::has_decomposed_cross_attention_sdpa(prefill_model)) { + m_kvcache_desc.max_prompt_size = whisper_kvcache_size - 1; + } + ov::npuw::util::PrepareWhisperPrefillModel(m_kvcache_desc.max_prompt_size, whisper_lhs_seq_size) .run_on_model(prefill_model); // Whisper decoder model ov::npuw::util::PrepareWhisperKVCacheModel().run_on_model(kvcache_model); // Whisper decoder_with_past model diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp index 3166017a8631ae..1e5be2b3edbaec 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp @@ -643,3 +643,7 @@ bool ov::npuw::util::PrepareWhisperKVCacheModel::run_on_model(const std::shared_ return true; } + +bool ov::npuw::util::has_decomposed_cross_attention_sdpa(const std::shared_ptr& model) { + return !find_decomposed_cross_attn_score_nodes(model).empty(); +} diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp index 69265be42d4923..d6fced6e8d35ba 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.hpp @@ -34,6 +34,12 @@ class PrepareWhisperKVCacheModel : public ov::pass::ModelPass { bool run_on_model(const std::shared_ptr& model) override; }; +// True if cross-attention SDPA in this model has already been decomposed (word-level +// timestamps requested), detected from the "cross_attention_qk_scaled_scores" tensor name +// the decomposition attaches - there's no NPUW option for this anymore, GenAI decomposes +// upfront for every device. +bool has_decomposed_cross_attention_sdpa(const std::shared_ptr& model); + } // namespace util } // namespace npuw } // namespace ov diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp index 8218069bb31b35..a7571be50c910b 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/whisper_infer_request.cpp @@ -58,12 +58,7 @@ void ov::npuw::WhisperInferRequest::infer_prefill(ov::SoPtr input_i 0u, m_npuw_llm_compiled_model->m_kvcache_desc.num_stored_tokens); - // For word-level timestamps: the prefill submodel exposes one - // "cross_attention_qk_scaled_scores_N" output per decoder layer (GenAI decomposes - // cross-attention SDPA and adds these outputs before NPUW ever sees the model), found - // by name rather than by a precomputed count since NPUW no longer decomposes anything - // itself. Cache them here since get_tensor() is called per-port and cross-attention - // (unlike self-attention) only runs during prefill. + // For word-level timestamps. for (const auto& [name, port] : m_prefill_out_ports) { if (port.get_names().count(WhisperInferRequest::whisper_layer_names::qk_scores) > 0) { m_alignment_tensors.insert({name, m_prefill_request->get_tensor(port)}); From 59e9f0f918ee4d44468c3867f719e88e69c8591b Mon Sep 17 00:00:00 2001 From: Parth Maheshwari Date: Thu, 20 Aug 2026 13:42:40 +0100 Subject: [PATCH 3/4] [NPUW][Whisper] Guard against nameless ports in KV-cache name normalization Decomposed cross-attention (word-level timestamps) exposes qk-score outputs that may not carry a tensor name by the time normalize_*_key_value_names runs. get_any_name() throws on those; skip nameless ports before calling it. --- .../src/plugin/npuw/whisper/prepare_whisper_model.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp index 1e5be2b3edbaec..33a166df7d5b8d 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/whisper/prepare_whisper_model.cpp @@ -547,7 +547,10 @@ void expose_runtime_states_as_inputs(const std::shared_ptr& model) { void normalize_input_key_value_names(const std::shared_ptr& model) { ov::ResultVector new_results, old_results; for (const auto& in : model->inputs()) { - if (in.get_any_name().find("decoder") == std::string::npos) { + // Decomposed cross-attention (word-level timestamps) exposes extra ports + // (e.g. the qk-score outputs) that may have no tensor name at all here - + // get_any_name() throws on those, unlike get_names(). + if (in.get_names().empty() || in.get_any_name().find("decoder") == std::string::npos) { continue; } @@ -563,7 +566,9 @@ void normalize_input_key_value_names(const std::shared_ptr& model) { void normalize_output_key_value_names(const std::shared_ptr& model) { ov::ResultVector new_results, old_results; for (const auto& out : model->outputs()) { - if (out.get_any_name().find("decoder") == std::string::npos) { + // See normalize_input_key_value_names: decomposed cross-attention adds + // outputs (qk scores) that can be nameless at this point in the pipeline. + if (out.get_names().empty() || out.get_any_name().find("decoder") == std::string::npos) { continue; } From 4b5c28951bc5d233c9ac25dedb1b8ca8c754e8a3 Mon Sep 17 00:00:00 2001 From: Parth Maheshwari Date: Thu, 20 Aug 2026 15:40:36 +0100 Subject: [PATCH 4/4] [NPUW] Guard against nameless ports in KV-cache precision conversion cvt_kvcache_to_low_precision() iterates every model input/output and calls get_any_name() unconditionally. Decomposed cross-attention (word-level timestamps) adds ports that may have no tensor name at this point, causing this to throw before the isPastKeyValues*/ isPresentKeyValues* checks can skip them. --- .../convert_kvcache_to_precision.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/plugins/intel_npu/src/plugin/npuw/npuw_transformations/convert_kvcache_to_precision.cpp b/src/plugins/intel_npu/src/plugin/npuw/npuw_transformations/convert_kvcache_to_precision.cpp index 720336a65a85d4..888c727f772f3c 100644 --- a/src/plugins/intel_npu/src/plugin/npuw/npuw_transformations/convert_kvcache_to_precision.cpp +++ b/src/plugins/intel_npu/src/plugin/npuw/npuw_transformations/convert_kvcache_to_precision.cpp @@ -82,6 +82,11 @@ std::shared_ptr cvt_kvcache_to_low_precision(const std::shared_ptrinputs()) { + // Decomposed cross-attention (word-level timestamps) can add ports with no + // tensor name at all - get_any_name() throws on those, unlike get_names(). + if (tensor.get_names().empty()) { + continue; + } const auto& name = tensor.get_any_name(); if (ov::npuw::util::isPastKeyValuesKey(name).has_value()) { ppp.input(name).tensor().set_element_type(key_storage_type); @@ -91,6 +96,10 @@ std::shared_ptr cvt_kvcache_to_low_precision(const std::shared_ptroutputs()) { + // See above: skip nameless outputs (e.g. qk-score outputs) before get_any_name(). + if (tensor.get_names().empty()) { + continue; + } const auto& name = tensor.get_any_name(); if (ov::npuw::util::isPresentKeyValuesKey(name).has_value()) { ppp.output(name).tensor().set_element_type(key_storage_type);