Xiong/gemma4 12 b 0814

src/plugins/intel_cpu/src/nodes/executors/x64/subgraph.hpp

@@ -29,13 +29,7 @@ class SubgraphExecutor : public SubgraphRepackingExecutor<BrgemmCopyBKernel> {
                      const BufferScratchpadAllocator& allocator,
                      const ov::intel_cpu::MultiCacheWeakPtr& kernel_cache);
 
-#ifdef SNIPPETS_DEBUG_CAPS
-protected:
-    void segfault_detector() const override;
 
-private:
-    bool enabled_segfault_detector = false;
-#endif
 };
 
 class SubgraphStaticExecutor : public SubgraphRepackingStaticExecutor<SubgraphExecutor> {

src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc

@@ -73,6 +73,8 @@ INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_PREFIX_CACHING_MAX_NUM_BLOCKS, uint64_t, 128,
 INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_ENABLE_CONTINUOUS_PREFILL, bool, false, ov::intel_npu::npuw::llm, enable_continuous_prefill, "NPUW_LLM_ENABLE_CONTINUOUS_PREFILL", LLM, EXPOSED, CACHED, ALL)
 INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_CACHE_ROPE, bool, true, ov::intel_npu::npuw::llm, cache_rope, "NPUW_LLM_CACHE_ROPE", LLM, EXPOSED, CACHED, ALL)
 INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_ENABLE_BLOCK_BASED_KV_CACHE, bool, false, ov::intel_npu::npuw::llm, enable_block_based_kv_cache, "NPUW_LLM_ENABLE_BLOCK_BASED_KV_CACHE", LLM, EXPOSED, CACHED, ALL)
+INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_SLIDING_WINDOW, uint64_t, 0, ov::intel_npu::npuw::llm, sliding_window, "NPUW_LLM_SLIDING_WINDOW", LLM, EXPOSED, CACHED, ALL)
+INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_LAYER_TYPES, std::string, "", ov::intel_npu::npuw::llm, layer_types, "NPUW_LLM_LAYER_TYPES", LLM, EXPOSED, CACHED, ALL)
 INTEL_NPU_NPUW_STRING_ENUM_OPT(NPUW_LLM_PREFILL_MOE_HINT, ::intel_npu::npuw::llm::MoEHint, MoEHintOptionTraits, ov::intel_npu::npuw::llm, prefill_moe_hint, "NPUW_LLM_PREFILL_MOE_HINT", LLM, EXPOSED, CACHED, ALL)
 INTEL_NPU_NPUW_STRING_ENUM_OPT(NPUW_LLM_GENERATE_MOE_HINT, ::intel_npu::npuw::llm::MoEHint, MoEHintOptionTraits, ov::intel_npu::npuw::llm, generate_moe_hint, "NPUW_LLM_GENERATE_MOE_HINT", LLM, EXPOSED, CACHED, ALL)
 INTEL_NPU_NPUW_STRING_ENUM_OPT(NPUW_LLM_PREFILL_HINT, ::intel_npu::npuw::llm::PrefillHint, PrefillHintOptionTraits, ov::intel_npu::npuw::llm, prefill_hint, "NPUW_LLM_PREFILL_HINT", LLM, EXPOSED, CACHED, ALL)

src/plugins/intel_npu/src/plugin/npuw/attn/attn_subgraph.cpp

@@ -882,30 +882,43 @@ ov::npuw::v1::subgraphs::RuntimeBehaviorFactory make_runtime_factory() {
 
                         OPENVINO_ASSERT(hfa_desc->is_valid(), "HFA configuration must be valid");
                         const int64_t tile_size = hfa_desc->_tile_size;
-                        const int64_t total_kv_length = state.hfa_selector->context_length();
-                        const int64_t num_tiles = total_kv_length / tile_size;
-                        OPENVINO_ASSERT(total_kv_length % tile_size == 0,
-                                        "HFA total KV length must be multiple of tile size for now");
 
                         const auto& hfa_inputs = io.inputs;
                         const auto& sdpa_info = hfa_desc->_sdpa_attention_info;
                         const auto& sdpa_in = sdpa_info._sdpa_indices;
+                        const uint32_t K_SEQ_DIM = static_cast<uint32_t>(sdpa_info._k_seq_dim);
+                        const uint32_t V_SEQ_DIM = static_cast<uint32_t>(sdpa_info._v_seq_dim);
 
-                        // Collect all KV block tensors (works for single-block and multi-block cases)
+                        // Collect all KV block tensors and accumulate block_sum in one pass.
+                        // (works for single-block and multi-block cases)
                         NPUW_ASSERT(!sdpa_in.past_key_blocks.empty() && !sdpa_in.past_value_blocks.empty() &&
                                     "SDPA indices must have at least one past_key/value block");
                         NPUW_ASSERT(sdpa_in.past_key_blocks.size() == sdpa_in.past_value_blocks.size() &&
                                     "Number of past key blocks must match number of past value blocks");
                         std::vector<ov::SoPtr<ov::ITensor>> past_key_blocks;
                         std::vector<ov::SoPtr<ov::ITensor>> past_value_blocks;
+                        int64_t block_sum = 0;
                         for (size_t i = 0; i < sdpa_in.past_key_blocks.size(); ++i) {
                             past_key_blocks.push_back(hfa_inputs.at(sdpa_in.past_key_blocks[i]));
                             past_value_blocks.push_back(hfa_inputs.at(sdpa_in.past_value_blocks[i]));
+                            block_sum += static_cast<int64_t>(past_key_blocks.back()->get_shape()[K_SEQ_DIM]);
                         }
                         auto query_tensor = hfa_inputs.at(sdpa_in.query);
                         auto present_key_tensor = hfa_inputs.at(sdpa_in.present_key);
                         auto attention_mask_tensor = hfa_inputs.at(sdpa_in.attention_mask);
                         auto present_value_tensor = hfa_inputs.at(sdpa_in.present_value);
+                        block_sum += static_cast<int64_t>(present_key_tensor->get_shape()[K_SEQ_DIM]);
+
+                        // total_kv_length = min(context_length(), block_sum) works for both:
+                        //   Global SDPA: block_sum == context_length() → min picks either.
+                        //   SWA within window: block_sum == context_length() → same.
+                        //   SWA past window:   block_sum < context_length() (blocks capped at window_size)
+                        //                      → min picks block_sum, the actual occupied KV length.
+                        const int64_t total_kv_length = std::min(state.hfa_selector->context_length(), block_sum);
+                        const int64_t num_tiles = total_kv_length / tile_size;
+                        OPENVINO_ASSERT(total_kv_length % tile_size == 0,
+                                        "HFA total KV length must be multiple of tile size for now");
+
                         auto& regular_tile_request = state.hfa_requests.infer_requests[HFARequestSet::REGULAR_TILE];
                         auto& final_tile_request = state.hfa_requests.infer_requests[HFARequestSet::FINAL_TILE];
                         auto attention_output_tensor =
@@ -956,8 +969,6 @@ ov::npuw::v1::subgraphs::RuntimeBehaviorFactory make_runtime_factory() {
                         final_tile_request->set_tensor(hfa_desc->_compiled_final_tile_model->outputs()[0],
                                                        attention_output_tensor);
 
-                        const uint32_t K_SEQ_DIM = static_cast<uint32_t>(sdpa_info._k_seq_dim);
-                        const uint32_t V_SEQ_DIM = static_cast<uint32_t>(sdpa_info._v_seq_dim);
                         constexpr uint32_t MASK_KV_SEQ_DIM = 3;
                         size_t next_available_mask_buffer_idx = 0;
 

src/plugins/intel_npu/src/plugin/npuw/host_flash_attention.cpp

@@ -11,6 +11,7 @@
 
 #include "intel_npu/ops/flash_attention_tile.hpp"
 #include "logging.hpp"
+#include "npuw_transformations/detect_causal_mask.hpp"
 #include "openvino/core/validation_util.hpp"
 #include "openvino/op/ops.hpp"
 #include "openvino/openvino.hpp"
@@ -659,6 +660,10 @@ static std::shared_ptr<ov::Model> create_hfa_tile_model(const ov::Shape& q_shape
     // enable_mask_skipping is true (depending on the model mask type).
     // For the non-fused operation all tiles require mask
     const bool use_mask = is_final_tile || !fused_flash_attention || !enable_mask_skipping;
+    LOG_INFO("[HFA] " << (is_final_tile ? "FINAL" : "regular") << " tile mask decision: use_mask=" << use_mask
+                       << " (is_final_tile=" << is_final_tile
+                       << ", fused_flash_attention=" << fused_flash_attention
+                       << ", enable_mask_skipping=" << enable_mask_skipping << ")");
     auto f32_nodes = convert_inputs_to_f32(inputs, mask_dtype, compute_dtype, use_mask);
 
     FlashAttentionResults results;
@@ -977,6 +982,9 @@ std::optional<HostFlashAttention> HostFlashAttention::from(const std::shared_ptr
                                                            bool enable_mask_skipping) {
     LOG_INFO("Attempting to create HostFlashAttention"
              << (fused_flash_attention ? " with fused flash attention node" : ""));
+    LOG_INFO("[HFA] enable_mask_skipping=" << enable_mask_skipping
+                                            << " (regular, non-final tiles will "
+                                            << (enable_mask_skipping ? "SKIP" : "KEEP") << " the explicit mask)");
     LOG_BLOCK();
 
     // ========================================================================
@@ -1106,6 +1114,32 @@ std::optional<HostFlashAttention> HostFlashAttention::from(const std::shared_ptr
     LOG_INFO("Creating HFA tile models: tile_size=" << query_size << ", v_transposed=" << v_transposed
                                                     << ", block_kv=" << block_kv_dtype
                                                     << ", present_kv=" << present_kv_dtype << ", q=" << q_dtype);
+
+    // Per-SDPA mask-skipping override
+    // AnnotatePerSDPAMaskType may have annotated this subgraph's Add(QK, mask) node
+    // with its individual mask type. For mixed SWA + global-attention models
+    // (e.g. Gemma-4 E2B/E4B), each ATTN subgraph decides independently:
+    //
+    //   Causal: force enable_mask_skipping = true
+    //   SlidingWindow / no annotation: keep the global enable_mask_skipping unchanged
+    //
+    // SlidingWindow is not forced to false because the global flag already handles the
+    // window_size >= max_prompt_len case (wide SWA that covers the full context).
+    bool local_enable_mask_skipping = enable_mask_skipping;
+    if (pattern_nodes.add_node) {
+        const auto& rt_info = pattern_nodes.add_node->get_rt_info();
+        const auto it = rt_info.find(ov::npuw::NPUW_SDPA_MASK_TYPE_RT_KEY);
+        if (it != rt_info.end()) {
+            const auto per_sdpa_mask_type = static_cast<ov::npuw::MaskInfo::MaskType>(it->second.as<int>());
+            if (per_sdpa_mask_type == ov::npuw::MaskInfo::MaskType::Causal) {
+                local_enable_mask_skipping = true;
+                LOG_DEBUG("Per-SDPA mask annotation: Causal → mask skipping ENABLED for this ATTN subgraph");
+            } else {
+                LOG_DEBUG("Per-SDPA mask annotation: SlidingWindow/Unknown → use global mask skipping setting ("
+                          << (enable_mask_skipping ? "YES" : "NO") << ") for this ATTN subgraph");
+            }
+        }
+    }
     auto tile_model = create_hfa_tile_model(q_shape_static,
                                             block_kv_dtype,  // state_dtype
                                             block_kv_dtype,  // kv_tile_dtype (past blocks)
@@ -1115,7 +1149,7 @@ std::optional<HostFlashAttention> HostFlashAttention::from(const std::shared_ptr
                                             kv_num_heads,
                                             false,
                                             fused_flash_attention,
-                                            enable_mask_skipping,
+                                            local_enable_mask_skipping,
                                             v_transposed);
     if (!tile_model) {
         LOG_WARN("Failed to create HFA tile model");
@@ -1131,7 +1165,7 @@ std::optional<HostFlashAttention> HostFlashAttention::from(const std::shared_ptr
                                                   kv_num_heads,
                                                   true,
                                                   fused_flash_attention,
-                                                  enable_mask_skipping,
+                                                  local_enable_mask_skipping,
                                                   v_transposed,
                                                   output_dtype);
     if (!final_tile_model) {

src/plugins/intel_npu/src/plugin/npuw/infer_request_utils.cpp

@@ -4,6 +4,7 @@
 
 #include "infer_request_utils.hpp"
 
+#include <algorithm>
 #include <limits>
 
 #include "logging.hpp"
@@ -150,6 +151,125 @@ void ov::npuw::util::copy_tensor_by_dim(ov::SoPtr<ov::ITensor> src_tensor,
     }
 }
 
+void ov::npuw::util::write_kv_slice_sliding(ov::SoPtr<ov::ITensor> dst_tensor,
+                                            ov::SoPtr<ov::ITensor> src_new_kv,
+                                            uint32_t dst_kv_dim,
+                                            uint32_t src_kv_dim,
+                                            uint32_t num_stored_tokens_before,
+                                            uint32_t num_new_tokens,
+                                            SlidingBufferLayout layout) {
+    const uint32_t capacity = static_cast<uint32_t>(dst_tensor->get_shape()[dst_kv_dim]);
+    const uint32_t old_total = num_stored_tokens_before;
+    const uint32_t new_total = old_total + num_new_tokens;
+    const uint32_t old_valid = std::min(old_total, capacity);
+    const uint32_t new_valid = std::min(new_total, capacity);
+
+    // Clamp against the source's own length too: a source tensor may legitimately hold
+    // fewer valid tokens than `num_new_tokens` claims (e.g. when re-using another
+    // layer's already-capacity-limited past buffer as a source, see the header comment).
+    const uint32_t src_len = static_cast<uint32_t>(src_new_kv->get_shape()[src_kv_dim]);
+    const uint32_t tokens_to_write = std::min({num_new_tokens, new_valid, src_len});
+
+    if (layout == SlidingBufferLayout::Circular) {
+        // No shift, ever: token at absolute position p always lives at physical index
+        // (p % capacity). See SlidingBufferLayout's doc comment in the header for why
+        // this is safe. Skip the leading tokens of this call that would be immediately
+        // overwritten later in the very same call (mirrors the LeftAligned clamp above).
+        if (tokens_to_write == 0) {
+            return;
+        }
+        const uint32_t first_new_abs_pos = num_stored_tokens_before + (num_new_tokens - tokens_to_write);
+        const uint32_t dst_start = first_new_abs_pos % capacity;
+
+        auto src_slice = (src_len > tokens_to_write)
+                             ? make_tensor_slice(src_new_kv, src_kv_dim, src_len - tokens_to_write, src_len)
+                             : src_new_kv;
+
+        if (dst_start + tokens_to_write <= capacity) {
+            // Single contiguous write - also covers the not-yet-saturated warm-up
+            // phase, where dst_start == first_new_abs_pos, i.e. a plain append.
+            auto dst_slice = make_tensor_slice(dst_tensor, dst_kv_dim, dst_start, dst_start + tokens_to_write);
+            copy_tensor_by_dim(src_slice, dst_slice, src_kv_dim, dst_kv_dim);
+        } else {
+            // Wraps past the end of the buffer: split into two contiguous legs.
+            const uint32_t first_leg_len = capacity - dst_start;
+            const uint32_t second_leg_len = tokens_to_write - first_leg_len;
+
+            auto src_first_leg = make_tensor_slice(src_slice, src_kv_dim, 0u, first_leg_len);
+            auto dst_first_leg = make_tensor_slice(dst_tensor, dst_kv_dim, dst_start, capacity);
+            copy_tensor_by_dim(src_first_leg, dst_first_leg, src_kv_dim, dst_kv_dim);
+
+            auto src_second_leg = make_tensor_slice(src_slice, src_kv_dim, first_leg_len, tokens_to_write);
+            auto dst_second_leg = make_tensor_slice(dst_tensor, dst_kv_dim, 0u, second_leg_len);
+            copy_tensor_by_dim(src_second_leg, dst_second_leg, src_kv_dim, dst_kv_dim);
+        }
+        return;
+    }
+
+    const uint32_t keep = new_valid - tokens_to_write;
+    const bool needs_shift = (keep > 0 && keep < old_valid);
+
+    if (needs_shift && dst_kv_dim == 3u) {
+        // Transposed-V layout (dst_kv_dim == 3): a partial-slice shift touches only
+        // `old_valid` of the `capacity` columns, but a dim-3 slice of a [1,C,H,W] tensor
+        // is non-contiguous, so both the read (old_tail->copy_to) and the write
+        // (copy_tensor_by_dim -> copy_columns_by_row_chunks) legs degrade into C*H
+        // individual small (per-token) memory transactions. When dst_tensor lives in
+        // NPU-resident remote memory, per-transaction latency (not bytes moved)
+        // dominates, and C*H can be in the thousands - this is the empirically
+        // dominant cost of the sliding-window KV update (~600ms/step on real HW).
+        //
+        // Since the *whole* (unsliced) buffer is fully contiguous, round-trip it as a
+        // single big contiguous transfer instead: one bulk device->CPU copy, a cheap
+        // in-CPU-memory shift (regular DRAM, C*H iterations here are negligible), then
+        // one bulk CPU->device copy back. This trades "only move what changed" for
+        // "always move `capacity` columns, but in O(1) device-memory transactions".
+        LOG_DEBUG("[SWA] Bulk-shifting KV buffer (dim=3): keeping last "
+                  << keep << " of " << old_valid << " old token(s), capacity=" << capacity);
+        auto whole_tmp = allocMem(dst_tensor->get_element_type(), dst_tensor->get_shape(), "CPU", nullptr);
+        dst_tensor->copy_to(whole_tmp._ptr);  // single bulk contiguous transfer
+
+        auto old_tail_cpu = make_tensor_slice(whole_tmp, dst_kv_dim, old_valid - keep, old_valid);
+        auto shift_tmp = allocMem(dst_tensor->get_element_type(), old_tail_cpu->get_shape(), "CPU", nullptr);
+        old_tail_cpu->copy_to(shift_tmp._ptr);  // CPU-to-CPU, cheap regardless of iteration count
+        auto dst_front_cpu = make_tensor_slice(whole_tmp, dst_kv_dim, 0u, keep);
+        copy_tensor_by_dim(shift_tmp, dst_front_cpu, dst_kv_dim, dst_kv_dim);  // CPU-to-CPU
+
+        if (tokens_to_write > 0) {
+            auto src_slice = (src_len > tokens_to_write)
+                                 ? make_tensor_slice(src_new_kv, src_kv_dim, src_len - tokens_to_write, src_len)
+                                 : src_new_kv;
+            auto dst_back_cpu = make_tensor_slice(whole_tmp, dst_kv_dim, keep, keep + tokens_to_write);
+            copy_tensor_by_dim(src_slice, dst_back_cpu, src_kv_dim, dst_kv_dim);
+        }
+
+        whole_tmp->copy_to(dst_tensor._ptr);  // single bulk contiguous transfer back
+        return;
+    }
+
+    if (needs_shift) {
+        // Sliding window is (re)saturated: shift the surviving tail of the old content to
+        // the front of the buffer. A temporary CPU snapshot is used because dst and the
+        // "old" region alias the same tensor, making a direct in-place copy unsafe.
+        LOG_DEBUG("[SWA] Shifting KV buffer: keeping last " << keep << " of " << old_valid << " old token(s), dim="
+                                                             << dst_kv_dim << ", capacity=" << capacity);
+        auto old_tail = make_tensor_slice(dst_tensor, dst_kv_dim, old_valid - keep, old_valid);
+        auto tmp = allocMem(dst_tensor->get_element_type(), old_tail->get_shape(), "CPU", nullptr);
+        old_tail->copy_to(tmp._ptr);
+        auto dst_front = make_tensor_slice(dst_tensor, dst_kv_dim, 0u, keep);
+        copy_tensor_by_dim(tmp, dst_front, dst_kv_dim, dst_kv_dim);
+    }
+
+    if (tokens_to_write == 0) {
+        return;
+    }
+    auto src_slice = (src_len > tokens_to_write)
+                         ? make_tensor_slice(src_new_kv, src_kv_dim, src_len - tokens_to_write, src_len)
+                         : src_new_kv;
+    auto dst_back = make_tensor_slice(dst_tensor, dst_kv_dim, keep, keep + tokens_to_write);
+    copy_tensor_by_dim(src_slice, dst_back, src_kv_dim, dst_kv_dim);
+}
+
 std::optional<ov::Output<const ov::Node>> ov::npuw::util::find_port_by_name(
     const std::vector<ov::Output<const ov::Node>>& ports,
     const std::string& name) {