feat(ds4): support qtype 105/106 across heterogeneous GPUs

.github/workflows/ci.yml

@@ -177,7 +177,9 @@ jobs:
             -DCMAKE_BUILD_TYPE=Release
           cmake --build build \
             --target test_flash_attn_sparse test_deepseek4_mmid_grouped_cuda \
-              test_deepseek4_unit \
+              test_deepseek4_unit test_rocmfp3_mix_registry \
+              test_rocmfp_mix_slice_matvec test_rocmfp_mix_gateup_glu \
+              test_ds4_mix_registry_teardown \
             -j"$(nproc)"
 
       - name: Run flash-attn sparse kernel test on the 3090
@@ -191,6 +193,11 @@ jobs:
       - name: Run DeepSeek4 graph-generation regression on the 3090
         run: ./server/build/test_deepseek4_unit
 
+      - name: Run mixed-quantization registry safety tests on the 3090
+        run: |
+          ctest --test-dir server/build --output-on-failure \
+            -R 'rocmfp3_mix_registry|rocmfp_mix_slice_matvec|rocmfp_mix_gateup_glu|ds4_mix_registry_teardown'
+
       # Optional model-backed end-to-end smoke (real spec-decode on the 3090),
       # disabled by default because it builds dflash_server and lazy-loads the
       # ~16 GB Qwen3.6-27B target + draft (~1-2 min). The weights are already
@@ -299,11 +306,13 @@ jobs:
           cmake --build "$RUNNER_TEMP/rocmfp-build" \
             --target test_rocmfp4 test_rocmfpx test_rocmfp4_hip_tail test_rocmfpx_mmq \
               test_deepseek4_mmid_grouped_cuda test_deepseek4_unit \
-              test_recurrent_snapshot test_server_unit \
+              test_recurrent_snapshot test_server_unit test_rocmfp3_mix_registry \
+              test_rocmfp_mix_slice_matvec test_rocmfp_mix_gateup_glu \
+              test_ds4_mix_registry_teardown \
             --parallel 8
           ctest --test-dir "$RUNNER_TEMP/rocmfp-build" \
             --output-on-failure \
-            -R 'rocmfp4_reference|rocmfpx_reference|rocmfp4_hip_tail|rocmfpx_mmq|deepseek4_mmid_grouped_cuda|deepseek4_unit|recurrent_snapshot|ChainRollbackPolicy'
+            -R 'rocmfp4_reference|rocmfpx_reference|rocmfp4_hip_tail|rocmfpx_mmq|deepseek4_mmid_grouped_cuda|deepseek4_unit|recurrent_snapshot|ChainRollbackPolicy|rocmfp3_mix_registry|rocmfp_mix_slice_matvec|rocmfp_mix_gateup_glu|ds4_mix_registry_teardown'
 
   build-windows:
     name: Build Windows (MSVC + CUDA, library + server targets)

harness/qualification/deepseek4/qualify_ds4_q5_amd.sh

@@ -32,6 +32,10 @@ case "$Q5_VERIFY" in
 esac
 FP4_Q5_X4_PLUS1="${FP4_Q5_X4_PLUS1:-auto}"
 CRITICAL_PATH_PLACEMENT="${CRITICAL_PATH_PLACEMENT:-0}"
+MAIN_TO_PEER_RATE_EXPLICIT=0
+if [[ -n "${MAIN_TO_PEER_RATE:-}" ]]; then
+    MAIN_TO_PEER_RATE_EXPLICIT=1
+fi
 MAIN_TO_PEER_RATE="${MAIN_TO_PEER_RATE:-3.4}"
 BALANCE_MIN_HOT="${BALANCE_MIN_HOT:-0}"
 EXPERT_BUDGET_MB="${EXPERT_BUDGET_MB:-13200}"
@@ -44,14 +48,12 @@ HASH_MODELS="${HASH_MODELS:-0}"
 CUDA_GRAPH_STATS_EVERY="${CUDA_GRAPH_STATS_EVERY:-200}"
 CUDA_DISABLE_GRAPHS_DEVICES="${CUDA_DISABLE_GRAPHS_DEVICES:-}"
 DYNAMIC_ROUTE_BALANCE="${DYNAMIC_ROUTE_BALANCE:-0}"
-DYNAMIC_MAIN_SLOTS="${DYNAMIC_MAIN_SLOTS:-3}"
+DYNAMIC_MAIN_SLOTS="${DYNAMIC_MAIN_SLOTS:-auto}"
 DYNAMIC_MAIN_SLOTS_X2="${DYNAMIC_MAIN_SLOTS_X2:-}"
 DYNAMIC_MAIN_SLOTS_X4="${DYNAMIC_MAIN_SLOTS_X4:-}"
-VERIFY_WIDTH=$((4 + Q5_VERIFY))
-RUN_ID="${RUN_ID:-ds4-q${VERIFY_WIDTH}-fr${FORCE_GRAPH_REPLAY}-direct${DIRECT_INDEXER_TOPK}-radix${BLOCK_RADIX_TOPK}-x4p1${FP4_Q5_X4_PLUS1}-cp${CRITICAL_PATH_PLACEMENT}-r${MAIN_TO_PEER_RATE}-$(date -u +%Y%m%dT%H%M%SZ)}"
+EXPERT_TOP_K="${EXPERT_TOP_K:-4}"
+RUN_ID="${RUN_ID:-}"
 OUT_ROOT="${OUT_ROOT:-$CHECKOUT/results/ds4_q5_context_qualification}"
-OUT_DIR="$OUT_ROOT/$RUN_ID"
-SERVER_LOG="$OUT_DIR/server.log"
 
 for executable in "$SERVER_BIN" "$TOKENIZER_HARNESS"; do
     if [[ ! -f "$executable" || ! -x "$executable" ]]; then
@@ -95,18 +97,42 @@ case "$DYNAMIC_ROUTE_BALANCE" in
     0|1) ;;
     *) echo "DYNAMIC_ROUTE_BALANCE must be 0 or 1" >&2; exit 2 ;;
 esac
-case "$DYNAMIC_MAIN_SLOTS" in
-    1|2|3|4|5|6) ;;
-    *) echo "DYNAMIC_MAIN_SLOTS must be an integer from 1 through 6" >&2; exit 2 ;;
-esac
-case "$DYNAMIC_MAIN_SLOTS_X2" in
-    ""|2|3|4|5|6|7|8|9|10|11|12) ;;
-    *) echo "DYNAMIC_MAIN_SLOTS_X2 must be empty or an integer from 2 through 12" >&2; exit 2 ;;
-esac
-case "$DYNAMIC_MAIN_SLOTS_X4" in
-    ""|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24) ;;
-    *) echo "DYNAMIC_MAIN_SLOTS_X4 must be empty or an integer from 4 through 24" >&2; exit 2 ;;
-esac
+if [[ ! "$EXPERT_TOP_K" =~ ^[1-9][0-9]*$ ]] || ((EXPERT_TOP_K > 6)); then
+    echo "EXPERT_TOP_K must be an integer from 1 through 6" >&2
+    exit 2
+fi
+if [[ "$DYNAMIC_MAIN_SLOTS" != auto ]] &&
+   { [[ ! "$DYNAMIC_MAIN_SLOTS" =~ ^[1-9][0-9]*$ ]] ||
+     ((DYNAMIC_MAIN_SLOTS > EXPERT_TOP_K)); }; then
+    echo "DYNAMIC_MAIN_SLOTS must be auto or an integer from 1 through EXPERT_TOP_K ($EXPERT_TOP_K)" >&2
+    exit 2
+fi
+if [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]] &&
+   { [[ ! "$DYNAMIC_MAIN_SLOTS_X2" =~ ^[1-9][0-9]*$ ]] ||
+     ((DYNAMIC_MAIN_SLOTS_X2 < 2 || DYNAMIC_MAIN_SLOTS_X2 > 2 * EXPERT_TOP_K)); }; then
+    echo "DYNAMIC_MAIN_SLOTS_X2 must be empty or an integer from 2 through $((2 * EXPERT_TOP_K))" >&2
+    exit 2
+fi
+if [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]] &&
+   { [[ ! "$DYNAMIC_MAIN_SLOTS_X4" =~ ^[1-9][0-9]*$ ]] ||
+     ((DYNAMIC_MAIN_SLOTS_X4 < 4 || DYNAMIC_MAIN_SLOTS_X4 > 4 * EXPERT_TOP_K)); }; then
+    echo "DYNAMIC_MAIN_SLOTS_X4 must be empty or an integer from 4 through $((4 * EXPERT_TOP_K))" >&2
+    exit 2
+fi
+explicit_route_quotas=0
+if [[ "$DYNAMIC_MAIN_SLOTS" != auto ]]; then
+    ((explicit_route_quotas += 1))
+fi
+if [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]]; then
+    ((explicit_route_quotas += 1))
+fi
+if [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]]; then
+    ((explicit_route_quotas += 1))
+fi
+if ((explicit_route_quotas > 1)); then
+    echo "set at most one dynamic main-slot quota" >&2
+    exit 2
+fi
 case "$FP4_Q5_X4_PLUS1" in
     auto|0|1) ;;
     *) echo "FP4_Q5_X4_PLUS1 must be auto, 0, or 1" >&2; exit 2 ;;
@@ -139,12 +165,6 @@ case "$HASH_MODELS" in
     0|1) ;;
     *) echo "HASH_MODELS must be 0 or 1" >&2; exit 2 ;;
 esac
-case "$RUN_ID" in
-    ""|.|..|*[!A-Za-z0-9._-]*)
-        echo "RUN_ID may contain only letters, numbers, dot, underscore, and hyphen" >&2
-        exit 2
-        ;;
-esac
 for numeric_setting in PORT MAX_CTX EXPERT_BUDGET_MB WARMUP RUNS MAX_TOKENS \
     VRAM_MONITOR_SECONDS CUDA_GRAPH_STATS_EVERY; do
     numeric_value="${!numeric_setting}"
@@ -178,6 +198,48 @@ for target in "${target_args[@]}"; do
     fi
 done
 
+DYNAMIC_BALANCE_ENV_NAME=""
+DYNAMIC_BALANCE_ENV_VALUE=""
+DYNAMIC_BALANCE_LABEL="off"
+if [[ "$DYNAMIC_ROUTE_BALANCE" == 1 ]]; then
+    if [[ "$DYNAMIC_MAIN_SLOTS" != auto ]]; then
+        DYNAMIC_BALANCE_ENV_NAME="DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS"
+        DYNAMIC_BALANCE_ENV_VALUE="$DYNAMIC_MAIN_SLOTS"
+        DYNAMIC_BALANCE_LABEL="s${DYNAMIC_MAIN_SLOTS}"
+    elif [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]]; then
+        DYNAMIC_BALANCE_ENV_NAME="DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X2"
+        DYNAMIC_BALANCE_ENV_VALUE="$DYNAMIC_MAIN_SLOTS_X2"
+        DYNAMIC_BALANCE_LABEL="s2x${DYNAMIC_MAIN_SLOTS_X2}"
+    elif [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]]; then
+        DYNAMIC_BALANCE_ENV_NAME="DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X4"
+        DYNAMIC_BALANCE_ENV_VALUE="$DYNAMIC_MAIN_SLOTS_X4"
+        DYNAMIC_BALANCE_LABEL="s4x${DYNAMIC_MAIN_SLOTS_X4}"
+    elif [[ "$EXPERT_TOP_K" == 4 && "$MAIN_TO_PEER_RATE_EXPLICIT" == 0 ]]; then
+        # Preserve the qualified top-4 default. Automatic rate-based scaling is
+        # for widened top-k or an explicit operator override.
+        DYNAMIC_BALANCE_ENV_NAME="DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS"
+        DYNAMIC_BALANCE_ENV_VALUE=3
+        DYNAMIC_BALANCE_LABEL="s3"
+    else
+        DYNAMIC_BALANCE_ENV_NAME="DFLASH_MOE_TP_MAIN_TO_PEER_RATE"
+        DYNAMIC_BALANCE_ENV_VALUE="$MAIN_TO_PEER_RATE"
+        DYNAMIC_BALANCE_LABEL="r${MAIN_TO_PEER_RATE}"
+    fi
+fi
+
+VERIFY_WIDTH=$((4 + Q5_VERIFY))
+if [[ -z "$RUN_ID" ]]; then
+    RUN_ID="ds4-q${VERIFY_WIDTH}-k${EXPERT_TOP_K}-fr${FORCE_GRAPH_REPLAY}-direct${DIRECT_INDEXER_TOPK}-radix${BLOCK_RADIX_TOPK}-x4p1${FP4_Q5_X4_PLUS1}-cp${CRITICAL_PATH_PLACEMENT}-bal${DYNAMIC_BALANCE_LABEL}-$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+case "$RUN_ID" in
+    .|..|*[!A-Za-z0-9._-]*)
+        echo "RUN_ID may contain only letters, numbers, dot, underscore, and hyphen" >&2
+        exit 2
+        ;;
+esac
+OUT_DIR="$OUT_ROOT/$RUN_ID"
+SERVER_LOG="$OUT_DIR/server.log"
+
 # The script changes physical cards 0 and 1 with rocm-smi. A visibility mask
 # that reorders those cards would apply the performance levels to the wrong
 # logical devices, so this qualification only accepts the canonical order.
@@ -279,7 +341,7 @@ server_env=(
     "GGML_CUDA_GRAPH_STATS=1"
     "GGML_CUDA_GRAPH_STATS_EVERY=$CUDA_GRAPH_STATS_EVERY"
     "LUCE_CUDA_I32_REPEAT=1"
-    "DFLASH_DS4_TOPK=4"
+    "DFLASH_DS4_TOPK=$EXPERT_TOP_K"
     "DFLASH_DS4_FUSED_VERIFY=1"
     "DFLASH_DS4_FUSED_HYBRID_DECODE=1"
     "DFLASH_DS4_TIMING=1"
@@ -343,19 +405,9 @@ if [[ -n "$DECODE_HOTNESS_CSV" ]]; then
 fi
 if [[ "$DYNAMIC_ROUTE_BALANCE" == 1 ]]; then
     server_env+=(
-        "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE=1"
-        "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS=$DYNAMIC_MAIN_SLOTS"
+        "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE=1"
+        "$DYNAMIC_BALANCE_ENV_NAME=$DYNAMIC_BALANCE_ENV_VALUE"
     )
-    if [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]]; then
-        server_env+=(
-            "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS_X2=$DYNAMIC_MAIN_SLOTS_X2"
-        )
-    fi
-    if [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]]; then
-        server_env+=(
-            "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS_X4=$DYNAMIC_MAIN_SLOTS_X4"
-        )
-    fi
 fi
 if [[ -n "$CUDA_DISABLE_GRAPHS_DEVICES" ]]; then
     server_env+=(
@@ -417,7 +469,7 @@ server_args=(
     --hard-limit-reply-budget 0
     --chunk 2048
     --ds4-fused-decode
-    --ds4-expert-top-k 4
+    --ds4-expert-top-k "$EXPERT_TOP_K"
     --ds4-prefill sparse
     --peer-access
 )
@@ -442,6 +494,7 @@ server_args=(
     echo "dynamic_main_slots=$DYNAMIC_MAIN_SLOTS"
     echo "dynamic_main_slots_x2=$DYNAMIC_MAIN_SLOTS_X2"
     echo "dynamic_main_slots_x4=$DYNAMIC_MAIN_SLOTS_X4"
+    echo "expert_top_k=$EXPERT_TOP_K"
     echo "cache_slots=$CACHE_SLOTS"
     echo "mmvq_max_ncols=$MMVQ_MAX_NCOLS"
     echo "targets=$TARGETS"

server/CMakeLists.txt

@@ -1106,6 +1106,7 @@ if(DFLASH27B_TESTS)
         endif()
         target_include_directories(test_rocmfp3_mix_registry PRIVATE
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src/ggml-cuda)
         target_link_libraries(test_rocmfp3_mix_registry PRIVATE
             ggml ggml-base ${DFLASH27B_GGML_BACKEND_TARGET})
@@ -1132,6 +1133,7 @@ if(DFLASH27B_TESTS)
         endif()
         target_include_directories(test_rocmfp_mix_slice_matvec PRIVATE
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src/ggml-cuda)
         target_link_libraries(test_rocmfp_mix_slice_matvec PRIVATE
             ggml ggml-base ${DFLASH27B_GGML_BACKEND_TARGET})
@@ -1158,6 +1160,7 @@ if(DFLASH27B_TESTS)
         endif()
         target_include_directories(test_rocmfp_mix_gateup_glu PRIVATE
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src
             ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src/ggml-cuda)
         target_link_libraries(test_rocmfp_mix_gateup_glu PRIVATE
             ggml ggml-base ${DFLASH27B_GGML_BACKEND_TARGET})

server/deps/llama.cpp/ggml/include/ggml-cuda.h

@@ -89,6 +89,19 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);
 GGML_BACKEND_API bool ggml_backend_cuda_topk_rows(const struct ggml_tensor * logits, int k,
                                                   float * probs_out, int32_t * ids_out);
 
+// Attach learned per-expert decode tables to a mixed-precision tensor. The
+// host variants copy the tables to the device that owns `base`. Call the
+// matching unregister function before releasing the tensor's backing buffer.
+// Returns false without registering when validation or device setup fails.
+GGML_BACKEND_API bool ggml_cuda_rocmfp3_mix_register_host(
+        const void * base, size_t expert_stride, int n_experts, int out, int in,
+        const void * codebooks_bf16_host, const uint8_t * modes_host);
+GGML_BACKEND_API bool ggml_cuda_rocmfp2_mix_register_host(
+        const void * base, size_t expert_stride, int n_experts, int out, int in,
+        const void * codebooks_bf16_host, const uint8_t * modes_host);
+GGML_BACKEND_API void ggml_cuda_rocmfp2_mix_unregister(const void * base);
+GGML_BACKEND_API void ggml_cuda_rocmfp3_mix_unregister(const void * base);
+
 #ifdef  __cplusplus
 }
 #endif

server/deps/llama.cpp/ggml/src/ggml-cuda/mmq.cu

@@ -5,6 +5,36 @@
 #include "rocmfp2_mix.cuh"
 #include "rocmfp3_mix.cuh"
 
+namespace {
+
+class mix_registry_dispatch_guard {
+public:
+    explicit mix_registry_dispatch_guard(ggml_type type) : type_(type) {
+        if (type_ == GGML_TYPE_Q2_1_ROCMFP2_MIX) {
+            ggml_cuda_rocmfp2_mix_registry_lock();
+        } else if (type_ == GGML_TYPE_Q3_1_ROCMFP3_MIX) {
+            ggml_cuda_rocmfp3_mix_registry_lock();
+        }
+    }
+
+    ~mix_registry_dispatch_guard() {
+        if (type_ == GGML_TYPE_Q2_1_ROCMFP2_MIX) {
+            ggml_cuda_rocmfp2_mix_registry_unlock();
+        } else if (type_ == GGML_TYPE_Q3_1_ROCMFP3_MIX) {
+            ggml_cuda_rocmfp3_mix_registry_unlock();
+        }
+    }
+
+    mix_registry_dispatch_guard(const mix_registry_dispatch_guard &) = delete;
+    mix_registry_dispatch_guard & operator=(
+        const mix_registry_dispatch_guard &) = delete;
+
+private:
+    ggml_type type_;
+};
+
+}  // namespace
+
 static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const bool is_mix_type =
         args.type_x == GGML_TYPE_Q2_1_ROCMFP2_MIX ||
@@ -137,6 +167,10 @@ static void ggml_cuda_mul_mat_q_impl(
     const float * src1_d = (const float *) src1->data;
     float       *  dst_d = (float       *)  dst->data;
 
+    // Keep mix side data alive until every MMQ launch using it is enqueued.
+    // Registry teardown takes the same lock and drains the owning device before
+    // freeing those buffers.
+    mix_registry_dispatch_guard mix_guard(src0->type);
     const void * mix_codebooks_raw = nullptr;
     const uint8_t * mix_modes = nullptr;
     if (src0->type == GGML_TYPE_Q2_1_ROCMFP2_MIX) {
@@ -448,6 +482,7 @@ void ggml_cuda_op_mul_mat_q(
     const bool use_stream_k = ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA)
                             || GGML_CUDA_CC_IS_CDNA(cc))
                             && src1_ncols == ne11;
+    mix_registry_dispatch_guard mix_guard(src0->type);
     const void * mix_codebooks_raw = nullptr;
     const uint8_t * mix_modes = nullptr;
     if (src0->type == GGML_TYPE_Q2_1_ROCMFP2_MIX) {