diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 0ad0b31fd..70c7a1033 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -373,6 +373,44 @@ sampler stay on the main backend while routed target experts execute on their configured owners. `--ds4-expert-top-k 4` remains a separate approximate policy; omit it to retain the model's default six routed experts. +### Silent AR fallback (check this before calling spec decode broken) + +The DSpark verifier is greedy-only, so the server routes a request to plain +autoregressive decode — with the drafter loaded and `DSpark spec-decode +ENABLED` already printed at startup — whenever any of these hold: + +- the effective sampler is non-greedy (`temperature > 0`, repetition or + presence/frequency penalties), +- the request carries budget stop tokens, +- the request forces AR explicitly. + +The sampler trap is the subtle one: when a request **omits** `temperature`, +the HTTP layer falls back to the model card's sampling defaults, and +`share/model_cards/deepseek-v4-flash-0731-rocmfpx.json` sets +`temperature: 1.0`. A temperature-less benchmark request therefore decodes +pure AR on any build that ships the card, while the same request engages +speculation on a tree without it — which looks exactly like a spec-decode +regression between the two binaries. It is not one; send +`"temperature": 0.0` explicitly. The tell in the log is `decode tokens=N +steps=N-1` (one step per token) instead of `[ds4-spec] gen=... steps=...` +lines; the server also now logs one `DSpark spec loaded but this request +decodes AR: ...` line naming the condition the first time it happens. + +Speculative throughput is acceptance-bound, and acceptance is workload-bound: +on the gfx1151 host, code/math prompts (accept 0.83–0.90, ~2.4 committed +tokens per verify step) measured 24–27 tok/s where open-ended prose (accept +0.64–0.69, ~1.1 committed per step) measured 16–18 tok/s from the same server +under the same flags. Judge a spec-decode number against the acceptance it +was measured at, not against the headline figure from a different prompt mix +(see the gfx1151 numbers below for the measured governor/top-k envelope). + +Measure on an otherwise idle box. Strix Halo decode is unified-memory +bandwidth bound, so an unrelated co-tenant compile on the same host dropped +the identical request from 18.2 to 3.8 tok/s — a 5x swing with no change to +the acceptance rate or the step count, which is what makes it easy to misread +as a model or kernel regression. Record `/proc/loadavg` alongside any tok/s +figure, and re-run anything anomalous before believing it. + ### Verifier graph-cache safety Every heterogeneous verifier slot owns a scheduler and its per-backend scratch diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 47c67a300..0595c364f 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1841,6 +1841,25 @@ GenerateResult DeepSeek4Backend::generate_from_state( // The DSpark verifier is greedy-only. Route sampling and penalties through // AR so the request's sampler contract is not silently ignored. const bool sampling_requires_ar = sampler_.needs_logit_processing(); + // A drafter was loaded and the operator asked for spec decode, but this + // request routes to AR anyway. Say why, once: the DS4 model card defaults + // temperature to 1.0, so a request that merely OMITS temperature lands + // here — the server then decodes pure AR while the startup log still says + // "spec-decode ENABLED", which reads as a spec-engagement regression. + if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && + (req.force_ar_decode || budget_requires_ar || sampling_requires_ar)) { + static bool warned = false; + if (!warned) { + warned = true; + std::fprintf(stderr, + "[deepseek4] DSpark spec loaded but this request decodes AR: " + "force_ar=%d stop_tokens=%d sampling=%d (temp=%.2f rep_pen=%.2f " + "freq_pen=%.2f pres_pen=%.2f; greedy needs temperature 0)\n", + req.force_ar_decode ? 1 : 0, budget_requires_ar ? 1 : 0, + sampling_requires_ar ? 1 : 0, sampler_.temp, + sampler_.rep_pen, sampler_.freq_pen, sampler_.pres_pen); + } + } if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { if (last_logits_.empty()) { diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 223254902..114fc8213 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6779,6 +6779,35 @@ bool deepseek4_step_layer_range( q5_verify_candidate) && verify_hooks && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); + // Fused verify has many preconditions and declining any of them is + // invisible: the request still decodes, still reports a healthy acceptance + // rate, and only the throughput differs. Name the failed condition once so + // a slow DSpark run can be attributed from the log instead of guessed at. + // (No run has yet tripped this on gfx1151 — it is here so that the next + // "spec decode is slow" report starts from evidence.) + if (ds4_fused_verify_enabled() && !fused_verify_candidate && + n_tokens >= 2 && layer_begin == 0 && is_last_shard) { + static bool warned = false; + if (!warned) { + warned = true; + std::fprintf(stderr, + "[deepseek4] DFLASH_DS4_FUSED_VERIFY=1 but fused verify is " + "inactive: n_tokens=%d (cap %d) verify_hooks=%d out_logits=%d " + "backend_gpu=%d moe_hybrid=%d expert_runtime=%d " + "materialized_cold=%d cold_backend_kind_gpu=%d " + "cold_backend_distinct=%d; verify falls back to the dense " + "full-expert path\n", + n_tokens, GGML_CUDA_DS4_MIX_MMV_MAX_TOKENS, + verify_hooks ? 1 : 0, out_logits ? 1 : 0, + ds4_backend_is_gpu(backend) ? 1 : 0, + moe_hybrid ? 1 : 0, expert_runtime ? 1 : 0, + moe_hybrid && moe_hybrid->materialized_cold_experts ? 1 : 0, + moe_hybrid && moe_hybrid->cold_backend_kind == + MoeHybridColdBackend::Gpu ? 1 : 0, + moe_hybrid && moe_hybrid->cold_backend && + moe_hybrid->cold_backend != backend ? 1 : 0); + } + } const bool heterogeneous_sparse_prefill = !fused_verify_candidate && moe_hybrid && cache.prefill_mode == PrefillAttentionMode::Sparse &&