feat(ds4): add monolithic Strix Halo concurrent serving

.github/workflows/ci.yml

@@ -79,12 +79,16 @@ jobs:
             -DCMAKE_BUILD_TYPE=Release
           cmake --build build --target \
             test_dflash test_generate test_flash_attn_sparse test_server_unit \
-            test_deepseek4_unit -j$(nproc)
+            test_deepseek4_unit test_feature_gate test_seq_slot_manager \
+            test_seq_engine_contract test_seq_batch_plan test_client_send_buffer \
+            test_deepseek4_page_layout test_deepseek4_paged_cache -j$(nproc)
 
       - name: Run C++ server unit tests
         run: |
           cd server/build
-          ctest --output-on-failure -R "server_unit|deepseek4_unit" --no-tests=error
+          ctest --output-on-failure \
+            -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|deepseek4_page_layout|deepseek4_paged_cache" \
+            --no-tests=error
 
       - name: Populate venv with cu128 torch + setuptools
         # First pass: install the workspace's default deps. dflash declares

README.md

@@ -350,7 +350,9 @@ When compression is on, the request path picks one of three modes automatically,
 | `DFLASH_PREFILL_CACHE_SLOTS=N` | `0` | Container-entrypoint equivalent of `--prefill-cache-slots`; the native binary itself uses the CLI flag. |
 | `--kv-cache-dir <path>` | — | Persist prefix cache to disk |
 | `--kv-cache-budget N` | — | On-disk cache size cap |
-| `--paged-attention` | off | Exact 16-token block-table decode for single-device Qwen3.6-27B AR; see [paged attention](optimizations/paged_attention/README.md) |
+| `--paged-attention` | off | Exact block-table attention for single-device Qwen3.6-27B and monolithic DeepSeek4 on Strix Halo; see [paged attention](optimizations/paged_attention/README.md) and [DeepSeek4](server/docs/DS4.md#strix-halo-concurrent-serving) |
+| `--max-concurrency N` | `1` | DeepSeek4 Strix Halo sequence slots. Values 2–16 enable paged attention automatically; Qwen remains single-sequence. |
+| `--kv-pool-tokens N` | `0` (auto) | Shared physical K/V capacity for concurrent paged serving. Requires `--max-concurrency` greater than 1. Zero derives capacity from available device memory; explicit values are rounded to whole 16-token blocks. |
 
 **Bounded KV residency (KVFlash)**
 
@@ -388,6 +390,7 @@ Pages the attention KV cache through a fixed pool of GPU slots; cold 64-token ch
 | `--draft-ipc-bin <path>` | — | Out-of-process draft binary (mixed CUDA/HIP) |
 | `--peer-access` | off | Enable P2P between target GPUs |
 | `--chunk N` | backend default | Prefill ubatch size |
+| `--admission-coalesce-ms N` | `20` | Idle-to-busy batching window for concurrent serving, from 0 to 1000 ms; `0` disables it. |
 | `--no-cors` | CORS on | Disable CORS headers |
 | `DFLASH_TARGET_GPU=N` | `0` | Env var equivalent of `--target-gpu` |
 | `DFLASH_DRAFT_GPU=N` | same as target | Env var equivalent of `--draft-gpu` |

server/CMakeLists.txt

@@ -147,6 +147,7 @@ elseif(DFLASH27B_GPU_BACKEND STREQUAL "hip")
     endif()
     set(GGML_HIP  ON  CACHE BOOL "" FORCE)
     set(GGML_HIP_RCCL OFF CACHE BOOL "" FORCE)
+    set(GGML_HIP_GRAPHS OFF CACHE BOOL "Enable experimental HIP graph replay")
     set(DFLASH27B_GGML_BACKEND_TARGET ggml-hip)
     set(DFLASH27B_HIP_ARCHITECTURES "" CACHE STRING "HIP GPU targets, e.g. gfx906;gfx1100")
     set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
@@ -391,6 +392,8 @@ add_library(dflash_common STATIC
     src/deepseek4/deepseek4_loader.cpp
     src/deepseek4/deepseek4_graph.cpp
     src/deepseek4/deepseek4_roctx.cpp
+    src/deepseek4/deepseek4_paged_cache.cpp
+    src/deepseek4/deepseek4_seq_engine.cpp
     src/deepseek4/deepseek4_backend.cpp
     src/deepseek4/deepseek4_daemon.cpp
     src/deepseek4/deepseek4_layer_split_adapter.cpp
@@ -423,6 +426,7 @@ add_library(dflash_common STATIC
     src/common/dflash_draft_kv.cpp
     src/common/dflash_spec_decode.cpp
     src/common/paged_kv_pool.cpp
+    src/common/concurrency/seq_slot_manager.cpp
     src/common/layer_split_backend.cpp
     src/common/layer_split_runtime.cpp
     src/qwen35/graph_builders.cpp
@@ -1365,6 +1369,55 @@ if(DFLASH27B_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src)
         list(APPEND _raw_unit_test_targets test_paged_kv_pool)
     endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_seq_slot_manager.cpp")
+        # Host-side slot bookkeeping test (concurrent serving): no GPU.
+        add_executable(test_seq_slot_manager
+            test/test_seq_slot_manager.cpp
+            src/common/concurrency/seq_slot_manager.cpp
+            src/common/paged_kv_pool.cpp)
+        target_include_directories(test_seq_slot_manager PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        list(APPEND _raw_unit_test_targets test_seq_slot_manager)
+    endif()
+
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_deepseek4_page_layout.cpp")
+        add_executable(test_deepseek4_page_layout
+            test/test_deepseek4_page_layout.cpp)
+        target_include_directories(test_deepseek4_page_layout PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/include)
+        list(APPEND _raw_unit_test_targets test_deepseek4_page_layout)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_deepseek4_paged_cache.cpp")
+        add_executable(test_deepseek4_paged_cache
+            test/test_deepseek4_paged_cache.cpp
+            src/deepseek4/deepseek4_paged_cache.cpp)
+        target_compile_definitions(test_deepseek4_paged_cache PRIVATE DFLASH_DS4_PLAN_ONLY=1)
+        target_include_directories(test_deepseek4_paged_cache PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        list(APPEND _raw_unit_test_targets test_deepseek4_paged_cache)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_seq_engine_contract.cpp")
+        # SeqEngine conformance checker + the fakes that prove it bites: no GPU.
+        add_executable(test_seq_engine_contract test/test_seq_engine_contract.cpp)
+        target_include_directories(test_seq_engine_contract PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${CMAKE_CURRENT_SOURCE_DIR}/test)
+        list(APPEND _raw_unit_test_targets test_seq_engine_contract)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_seq_batch_plan.cpp")
+        # Pure-host tests for model-neutral token-budget/FIFO planning.
+        add_executable(test_seq_batch_plan test/test_seq_batch_plan.cpp)
+        target_include_directories(test_seq_batch_plan PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${CMAKE_CURRENT_SOURCE_DIR}/test)
+        list(APPEND _raw_unit_test_targets test_seq_batch_plan)
+    endif()
+    if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_client_send_buffer.cpp")
+        # Buffered non-blocking client writer test (socketpair): no GPU.
+        add_executable(test_client_send_buffer test/test_client_send_buffer.cpp)
+        target_include_directories(test_client_send_buffer PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        list(APPEND _raw_unit_test_targets test_client_send_buffer)
+    endif()
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_dflash.cpp")
         add_executable(test_dflash test/test_dflash.cpp)
         target_include_directories(test_dflash PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
@@ -1494,6 +1547,7 @@ if(DFLASH27B_TESTS)
         add_executable(test_server_unit ${_server_unit_sources})
         target_sources(test_server_unit PRIVATE
             src/server/http_server.cpp
+            src/server/scheduler.cpp
             src/server/model_card.cpp
             src/server/prompt_normalize.cpp
             src/qwen3/anchor_scan.cpp)
@@ -1743,6 +1797,7 @@ if(DFLASH27B_SERVER)
         add_executable(dflash_server
             src/server/server_main.cpp
             src/server/http_server.cpp
+            src/server/scheduler.cpp
             src/server/model_card.cpp
             src/server/prompt_normalize.cpp
         )

server/docs/DS4.md

@@ -207,6 +207,43 @@ performance profile held 48.1 tok/s median on the deterministic 128-token
 workload. The all-6-expert reference-exact mode is a correctness profile, not
 a throughput profile.
 
+### Strix Halo concurrent serving
+
+DeepSeek4 paged concurrency is deliberately a single-device path: one local
+HIP target on Strix Halo (`gfx1151`), with the complete model and every expert
+resident on that device. It does not use layer splitting, CUDA/HIP expert
+ownership, host-streamed experts, or DSpark.
+
+The backend keeps raw MLA rows, compressed rows, indexer state, sequence
+lengths, and block tables in a persistent 128-token paged cache. The shared
+HTTP scheduler performs admission, cancellation, slow-client isolation, and
+fair continuous batching. DeepSeek4 lowers each scheduler plan into one exact
+gathered graph with up to 16 independent lanes. Decode rows share the weight
+pass; each selected prompt advances by one exact token because the graph must
+not contain two rows from the same sequence.
+
+```bash
+cmake -S . -B build-hip \
+  -DDFLASH27B_GPU_BACKEND=hip \
+  -DDFLASH27B_HIP_ARCHITECTURES=gfx1151 \
+  -DDFLASH27B_SERVER=ON
+cmake --build build-hip -j
+
+./build-hip/dflash_server /path/to/deepseek4-target.gguf \
+  --target-device hip:0 \
+  --paged-attention \
+  --max-concurrency 16 \
+  --kv-pool-tokens 8192 \
+  --max-ctx 4096 \
+  --ds4-prefill exact \
+  --prefix-cache-slots 0
+```
+
+This mode fails closed for non-gfx1151 devices, CUDA, layer or remote target
+splits, `DFLASH_DS4_MOE_TP`, drafts/DSpark, DDTree, PFlash/KVFlash, fused
+decode, approximate prefill, windowed attention, and prefix-cache parking.
+There is no automatic fallback to a slower or asymmetric execution mode.
+
 ### Local single-shard
 
 If the adapter decides all 43 layers fit on one CUDA GPU, it loads a single shard locally and no IPC daemon is involved.

server/share/status.html

@@ -197,7 +197,9 @@ <h2>Decode Performance</h2>
 
 function update(data) {
   const badge = document.getElementById('phase-badge');
-  badge.textContent = data.phase;
+  badge.textContent = data.active_requests
+    ? data.phase + ' (' + data.active_requests + ' active)'
+    : data.phase;
   badge.className = 'badge badge-' + data.phase;
   document.getElementById('total-req').textContent = data.total_requests;
 

server/src/common/backend_args.h

@@ -25,6 +25,11 @@ struct BackendFeatureConfig {
     // time rather than through BackendArgs.
     bool routing_stats_requested = false;    // --freq / --collect-routing
     bool adaptive_experts_requested = false; // --adaptive-experts
+
+    // A fixed KVFlash pool requested through DFLASH_KVFLASH. "auto" is
+    // resolved later by the backend because only it has the VRAM budget needed
+    // to know whether a pool will actually be active.
+    bool kvflash_enabled = false;
 };
 
 // A superset of all per-architecture config fields. The factory reads only
@@ -59,6 +64,12 @@ struct BackendArgs {
     // only the fields they support.
     int             fa_window        = 0;  // 0 = full attention. qwen3.6 full-attn layers must see the whole context; a finite window drops the system prompt/tools -> breaks tool calls.
     bool            paged_attention  = false;  // 16-token paged K/V blocks for AR decode
+    // Concurrent decode slots (--max-concurrency). > 1 requires paged_attention;
+    // the backend serves that many sequences through the seq_* slot API.
+    int             max_concurrency  = 1;
+    // Total paged K/V pool in tokens shared by all slots (--kv-pool-tokens;
+    // block-rounded). 0 = derive capacity from available device memory.
+    long long       kv_pool_tokens   = 0;
     int             kq_stride_pad    = 32;
     int             draft_swa_window = 0;
     int             draft_ctx_max    = 4096;

server/src/common/backend_factory.cpp

@@ -104,6 +104,9 @@ DFLASH_CHECK_ARCH("deepseek4", DeepSeek4BackendConfig, DeepSeek4LayerSplitAdapte
 // never reads a block table — which is why its capability row is Never.)
 DFLASH_CHECK_ARCH_OPTION("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig,
                          has_paged_attention, paged_attn);
+DFLASH_CHECK_ARCH_OPTION("deepseek4", DeepSeek4BackendConfig,
+                         DeepSeek4LayerSplitAdapterConfig,
+                         has_paged_attention, paged_attn);
 
 #undef DFLASH_CHECK_ARCH
 #undef DFLASH_CHECK_ARCH_OPTION
@@ -423,6 +426,9 @@ std::unique_ptr<ModelBackend> create_backend(
             cfg.expert_top_k = args.ds4_expert_top_k;
             cfg.fused_decode = args.ds4_fused_decode;
             cfg.prefill_mode = args.ds4_prefill_mode;
+            cfg.paged_attention = args.paged_attention;
+            cfg.max_concurrency = args.max_concurrency;
+            cfg.kv_pool_tokens = args.kv_pool_tokens;
 
             auto backend = std::make_unique<DeepSeek4Backend>(cfg);
             if (!backend->init()) {