feat(sdk): add MTP speculative-decoding coverage to bench matrix

.github/workflows/bench.yml

@@ -45,6 +45,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set.outputs.matrix }}
       devices: ${{ steps.set.outputs.devices }}
+      excludes: ${{ steps.set.outputs.excludes }}
     steps:
       - uses: actions/checkout@v7
       - id: set
@@ -64,24 +65,35 @@ jobs:
           fi
           echo "devices=$devices" >> "$GITHUB_OUTPUT"
 
+          # Catalog-only rows (empty `devices`) are never dispatched.
           if [ -n "$PICK_MODEL" ]; then
             matrix=$(jq -c --arg s "$PICK_MODEL" '
               ($s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))) as $pick
               | map(select(.name as $n | $pick | index($n)))
+              | map(select(.devices != []))
             ' "$MODELS_FILE")
             picked_n=$(jq 'length' <<<"$matrix")
             want_n=$(jq -n --arg s "$PICK_MODEL" \
               '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0)) | length')
             if [ "$picked_n" != "$want_n" ]; then
-              echo "::error::model input has unknown name(s): $PICK_MODEL"
-              echo "::error::available: $(jq -r '.[].name' "$MODELS_FILE" | paste -sd, -)"
+              echo "::error::model input has unknown or catalog-only name(s): $PICK_MODEL"
+              echo "::error::available: $(jq -r '.[] | select(.devices != []) | .name' "$MODELS_FILE" | paste -sd, -)"
               exit 1
             fi
           else
-            matrix=$(jq -c . "$MODELS_FILE")
+            matrix=$(jq -c '[.[] | select(.devices != [])]' "$MODELS_FILE")
           fi
           echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
 
+          # Spec-decoding rows only run on SC8480XP; drop the other combos.
+          excludes=$(jq -c -n --argjson devices "$devices" --argjson models "$matrix" '
+            [ $devices[] as $d | $models[] as $m
+              | select(($m.spec // null) != null and $d != "SC8480XP")
+              | {device: $d, model: {name: $m.name}}
+            ]
+          ')
+          echo "excludes=$excludes" >> "$GITHUB_OUTPUT"
+
   bench:
     name: ${{ matrix.device }} · ${{ matrix.model.plugin }} · ${{ matrix.model.name }}
     needs: [build-sdk, load-models]
@@ -93,6 +105,7 @@ jobs:
       matrix:
         device: ${{ fromJson(needs.load-models.outputs.devices) }}
         model: ${{ fromJson(needs.load-models.outputs.matrix) }}
+        exclude: ${{ fromJson(needs.load-models.outputs.excludes) }}
     env:
       QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
     steps:

sdk/benchmark/benchmark.c

@@ -89,6 +89,7 @@ typedef struct {
     char*       mm_model_path;
     char*       mm_mmproj;
     char*       mm_tokenizer;
+    char*       mm_draft_model;
     bool        force_vlm; /* run VLM path even without an mmproj (QAIRT bundles) */
     bool        mm_is_vlm; /* manager classified the resolved model as VLM (geniex_ModelType) */
     const char* image_paths[MAX_PATHS];
@@ -514,6 +515,63 @@ static int resolve_via_mm(options_t* o, const char* id_in) {
     return 0;
 }
 
+/* Without this the plugin gets a raw id like "org/repo:Q4_0", fails to
+ * open it as a file, and silently falls back to non-speculative decode. */
+static int resolve_draft_via_mm(options_t* o) {
+    if (!o->draft_model || looks_like_path(o->draft_model)) return 0;
+    if (!g_mm_inited) {
+        int32_t rc = geniex_model_init(o->mm_data_dir);
+        if (rc != GENIEX_SUCCESS) {
+            const char* m = geniex_model_last_error_message();
+            fprintf(stderr, "ERROR: geniex_model_init: %s (%d)\n", m ? m : "?", rc);
+            return 1;
+        }
+        g_mm_inited = true;
+    }
+    size_t n   = strlen(o->draft_model);
+    char*  buf = (char*)malloc(n + 1);
+    if (!buf) return 1;
+    memcpy(buf, o->draft_model, n + 1);
+    const char* name;
+    const char* quant;
+    split_id(buf, &name, &quant);
+    geniex_ModelPaths paths;
+    memset(&paths, 0, sizeof(paths));
+    int32_t rc = geniex_model_get_paths(o->draft_model, &paths);
+    if (rc != GENIEX_SUCCESS) {
+        geniex_ModelPullInput in;
+        memset(&in, 0, sizeof(in));
+        in.struct_size = (uint32_t)sizeof(in);
+        in.model_name  = name;
+        in.quant       = quant;
+        in.hub         = parse_hub(o->mm_hub);
+        in.chipset     = o->mm_chipset;
+        in.model_type  = GENIEX_MODEL_TYPE_AUTO;
+        fprintf(stderr, "[mm  ] pulling draft %s%s%s ...\n", name, quant ? ":" : "", quant ? quant : "");
+        rc = geniex_model_pull(&in);
+        if (rc != GENIEX_SUCCESS) {
+            const char* m = geniex_model_last_error_message();
+            fprintf(stderr, "ERROR: geniex_model_pull(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc);
+            free(buf);
+            return 1;
+        }
+        rc = geniex_model_get_paths(o->draft_model, &paths);
+        if (rc != GENIEX_SUCCESS) {
+            const char* m = geniex_model_last_error_message();
+            fprintf(stderr, "ERROR: geniex_model_get_paths(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc);
+            free(buf);
+            return 1;
+        }
+    }
+    free(buf);
+    o->mm_draft_model = paths.model_path;
+    paths.model_path  = NULL;
+    geniex_model_paths_free(&paths);
+    o->draft_model = o->mm_draft_model;
+    fprintf(stderr, "[mm  ] resolved draft %s -> %s\n", o->draft_model, o->mm_draft_model);
+    return 0;
+}
+
 /* If `path` is a directory, return a heap-allocated path to a regular file
  * inside it (preferring `tokenizer.json`, otherwise the lexicographically
  * first regular file). The SDK derives the model dir via `parent_path()`,
@@ -658,6 +716,7 @@ static void parse_args(int argc, char** argv, options_t* o) {
     o->mm_model_path      = NULL;
     o->mm_mmproj          = NULL;
     o->mm_tokenizer       = NULL;
+    o->mm_draft_model     = NULL;
     o->force_vlm          = false;
     o->mm_is_vlm          = false;
     o->image_count        = 0;
@@ -1529,7 +1588,7 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i
     fprintf(f, "    \"params\": {\n");
     fprintf(f,
         "      \"warmup\": %d, \"repetitions\": %d, \"n_prompt\": %d, \"n_gen\": %d,\n"
-        "      \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d\n",
+        "      \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d",
         o->warmup,
         o->repeat,
         o->n_prompt,
@@ -1539,7 +1598,16 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i
         o->n_ctx,
         o->n_threads,
         ngl);
-    fprintf(f, "    },\n");
+    if (o->spec_type) {
+        fprintf(f, ",\n      \"spec_type\": ");
+        json_write_quoted(f, o->spec_type);
+        if (o->draft_model) {
+            fprintf(f, ",\n      \"draft_model\": ");
+            json_write_quoted(f, o->draft_model);
+        }
+        fprintf(f, ",\n      \"draft_tokens\": %d", o->draft_tokens);
+    }
+    fprintf(f, "\n    },\n");
     fprintf(f, "    \"runs\": [\n");
     for (int i = 0; i < o->repeat; ++i) {
         const run_result_t* r = &runs[i];
@@ -1846,6 +1914,10 @@ static int run_one_cell(options_t* o) {
         o->force_vlm = true;
     }
 
+    if (resolve_draft_via_mm(o) != 0) {
+        return 1;
+    }
+
     char* anchored = resolve_local_anchor(o->model_path);
     if (anchored) {
         fprintf(stderr, "[info] resolved model dir to anchor: %s\n", anchored);
@@ -1912,6 +1984,10 @@ static int run_one_cell(options_t* o) {
             geniex_free(o->mm_tokenizer);
             o->mm_tokenizer = NULL;
         }
+        if (o->mm_draft_model) {
+            geniex_free(o->mm_draft_model);
+            o->mm_draft_model = NULL;
+        }
         return 0;
     }
 
@@ -1955,6 +2031,10 @@ static int run_one_cell(options_t* o) {
         geniex_free(o->mm_tokenizer);
         o->mm_tokenizer = NULL;
     }
+    if (o->mm_draft_model) {
+        geniex_free(o->mm_draft_model);
+        o->mm_draft_model = NULL;
+    }
     return 0;
 }
 

sdk/benchmark/qdc/bench-models.json

@@ -126,6 +126,28 @@
     "vlm": true,
     "image": true
   },
+  {
+    "name": "gemma-4-26B-A4B-it-assistant",
+    "plugin": "llama_cpp",
+    "devices": [],
+    "model_id": "RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf:Q4_0",
+    "hub": "hf",
+    "url": "https://huggingface.co/RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf/resolve/main/gemma-4-26b-A4B-it-assistant-Q4_0.gguf"
+  },
+  {
+    "name": "gemma-4-26B-A4B-it-mtp",
+    "plugin": "llama_cpp",
+    "devices": ["npu", "hybrid"],
+    "model_id": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf:Q4_0",
+    "hub": "hf",
+    "url": "https://huggingface.co/google/gemma-4-26B-A4B-it-qat-q4_0-gguf/resolve/main/gemma-4-26B_q4_0-it.gguf",
+    "ctx": [512, 1024, 4096, 8192],
+    "spec": {
+      "type": "draft-mtp",
+      "draft": "gemma-4-26B-A4B-it-assistant",
+      "n_max": 3
+    }
+  },
   {
     "name": "Qwen3.5-2B",
     "plugin": "llama_cpp",

sdk/benchmark/qdc/linux/run_linux.sh

@@ -59,7 +59,7 @@ for ctx in "${CTX_ARR[@]}"; do
   : > "/data/local/tmp/matrix-qairt-${ctx}.tsv"
 done
 
-while IFS='|' read -r name plugin devs model_id vlm image; do
+while IFS='|' read -r name plugin devs model_id vlm image _spec_type _draft_id _draft_tokens; do
   [ -z "$name" ] && continue
   echo "=== plan $name id=$model_id ==="
   case "$plugin" in