From 257bd8013319f47033b8e2f929eff5495fbf3de5 Mon Sep 17 00:00:00 2001 From: alisachen-google Date: Mon, 26 Oct 2026 02:15:14 +0000 Subject: [PATCH] Add A4X-MAX Dynamo Pareto-sweep recipes for DeepSeek-V4-Pro and DeepSeek-R1 Package every operating point of the InferenceX GB300 Pareto curves for DeepSeek-V4-Pro (MXFP4 + EAGLE MTP), DeepSeek-R1 NVFP4 and DeepSeek-R1 FP8 as deployable disaggregated Dynamo + SGLang recipes on A4X-MAX GKE: - inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/: README with three small architecture diagrams, per-model Pareto maps (points labelled low latency / mid-curve / max throughput), deploy and sa-bench 8k/1k benchmark steps, and links to every values file and serving config. - pareto-sweep/{deepseek-v4-pro,deepseek-r1-nvfp4,deepseek-r1-fp8}/: 14 Helm values files (one per topology, 2 to 18 nodes) with a short README per model. - src/frameworks/a4xmax/dynamo-configs/: 28 prefill/decode serving configs taken flag-for-flag from the srt-slurm recipes of each point. - src/docker/dynamo-sglang-deepseek/Dockerfile: layers a released ai-dynamo onto a released SGLang image (DeepSeek-V4-Pro: SGLang v0.5.13 + Dynamo 1.2.0; DeepSeek-R1: SGLang v0.5.8.post1 + Dynamo 0.8.x). It links the NIXL libraries only when present and warns when they are missing. - Parent a4xmax disaggregated README and top-level index link the sub-recipe. --- README.md | 2 + .../disaggregated-serving/dynamo/README.md | 12 + .../dynamo/pareto-sweep/README.md | 571 ++++++++++++++++++ .../pareto-sweep/deepseek-r1-fp8/README.md | 11 + .../deepseek-r1-fp8/dsr1_fp8_low_latency.yaml | 221 +++++++ .../dsr1_fp8_max_throughput.yaml | 221 +++++++ .../deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml | 221 +++++++ .../pareto-sweep/deepseek-r1-nvfp4/README.md | 11 + .../dsr1_fp4_low_latency.yaml | 221 +++++++ .../dsr1_fp4_max_throughput.yaml | 229 +++++++ .../deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml | 229 +++++++ .../pareto-sweep/deepseek-v4-pro/README.md | 16 + .../deepseek-v4-pro/dsv4_p1_low_latency.yaml | 223 +++++++ .../deepseek-v4-pro/dsv4_p2_low_latency.yaml | 233 +++++++ .../deepseek-v4-pro/dsv4_p3_mid_curve.yaml | 241 ++++++++ .../deepseek-v4-pro/dsv4_p4_mid_curve.yaml | 241 ++++++++ .../deepseek-v4-pro/dsv4_p5_mid_curve.yaml | 241 ++++++++ .../deepseek-v4-pro/dsv4_p6_mid_curve.yaml | 241 ++++++++ .../dsv4_p7_max_throughput.yaml | 241 ++++++++ .../dsv4_p8_max_throughput.yaml | 241 ++++++++ src/docker/dynamo-sglang-deepseek/Dockerfile | 58 ++ ...1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml | 45 ++ ...-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml | 48 ++ ...max-throughput-10p1d-tp4-dep32-decode.yaml | 58 ++ ...ax-throughput-10p1d-tp4-dep32-prefill.yaml | 50 ++ ...1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml | 58 ++ ...-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml | 50 ++ ...1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml | 46 ++ ...-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml | 48 ++ ...max-throughput-6p1d-dep8-dep24-decode.yaml | 55 ++ ...ax-throughput-6p1d-dep8-dep24-prefill.yaml | 54 ++ ...-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml | 55 ++ ...fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml | 54 ++ ...v4-p1-low-latency-1p1d-tp4-tp4-decode.yaml | 43 ++ ...4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml | 38 ++ ...4-p2-low-latency-1p6d-dep4-tp4-decode.yaml | 43 ++ ...-p2-low-latency-1p6d-dep4-tp4-prefill.yaml | 39 ++ ...v4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml | 45 ++ ...4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml | 40 ++ ...4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml | 45 ++ ...-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml | 40 ++ ...v4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml | 45 ++ ...4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml | 40 ++ ...v4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml | 45 ++ ...4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml | 40 ++ ...-max-throughput-6p1d-dep4-dep8-decode.yaml | 45 ++ ...max-throughput-6p1d-dep4-dep8-prefill.yaml | 40 ++ ...-max-throughput-8p1d-dep4-dep8-decode.yaml | 45 ++ ...max-throughput-8p1d-dep4-dep8-prefill.yaml | 40 ++ 49 files changed, 5219 insertions(+) create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/README.md create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_low_latency.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_max_throughput.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/README.md create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_low_latency.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_max_throughput.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/README.md create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p1_low_latency.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p2_low_latency.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p3_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p4_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p5_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p6_mid_curve.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p7_max_throughput.yaml create mode 100644 inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p8_max_throughput.yaml create mode 100644 src/docker/dynamo-sglang-deepseek/Dockerfile create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml create mode 100644 src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml diff --git a/README.md b/README.md index 4f52fdb1..d06c02af 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,8 @@ Models | GPU Machine Type | **Reflection Beam 501B** | [A4X MAX (NVIDIA GB300)](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms) | Dynamo + SGLang | Aggregated Inference | GKE | [Link](./inference/a4xmax/aggregated-serving/dynamo/README.md) | | **Reflection Beam 501B** | [A4X MAX (NVIDIA GB300)](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms) | Dynamo + SGLang | Disaggregated Inference | GKE | [Link](./inference/a4xmax/disaggregated-serving/dynamo/README.md) | | **DeepSeek R1 671B** | [A4X MAX (NVIDIA GB300)](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms) | Dynamo + SGLang | Disaggregated Inference | GKE | [Link](./inference/a4xmax/disaggregated-serving/dynamo/README.md) | +| **DeepSeek V4 Pro (MXFP4 + MTP)** | [A4X MAX (NVIDIA GB300)](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms) | Dynamo + SGLang | Disaggregated Inference (InferenceX Pareto sweep) | GKE | [Link](./inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md) | +| **DeepSeek R1 671B (NVFP4 / FP8)** | [A4X MAX (NVIDIA GB300)](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms) | Dynamo + SGLang | Disaggregated Inference (InferenceX Pareto sweep) | GKE | [Link](./inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md) | ### Inference benchmarks G4 diff --git a/inference/a4xmax/disaggregated-serving/dynamo/README.md b/inference/a4xmax/disaggregated-serving/dynamo/README.md index 14eacbfa..17eb2eb3 100644 --- a/inference/a4xmax/disaggregated-serving/dynamo/README.md +++ b/inference/a4xmax/disaggregated-serving/dynamo/README.md @@ -20,6 +20,7 @@ Dynamo provides a disaggregated architecture that separates prefill and decode o * [3.1. Supported Models and Deployment Categories](#supported-models) * [3.2. Category 1: High Interactivity / Low Latency Deployment](#sglang-high-interactivity) * [3.3. Category 2: High Throughput Multi-Node Deployment](#sglang-high-throughput) + * [3.4. Category 3: InferenceX Pareto-Sweep Operating Points (DeepSeek-V4-Pro, DeepSeek-R1)](#sglang-pareto-sweep) * [4. Inference Request and Benchmarking](#inference-request) * [4.1. Verify Deployment Readiness](#verify-readiness) * [4.2. Deploy the Benchmark Client](#deploy-bench-client) @@ -325,6 +326,17 @@ helm install -f ${VALUES_FILE} \ $REPO_ROOT/src/helm-charts/a4xmax/inference-templates/dynamo-deployment ``` + +### 3.4. Category 3: InferenceX Pareto-Sweep Operating Points (DeepSeek-V4-Pro, DeepSeek-R1) + +The [`pareto-sweep/`](./pareto-sweep/README.md) sub-recipe packages every operating point of the public InferenceX GB300 Pareto curves for **DeepSeek-V4-Pro (MXFP4 + EAGLE MTP)**, **DeepSeek-R1 (NVFP4)** and **DeepSeek-R1 (FP8)** as a values file plus a prefill/decode serving-config pair for this chart. Each point is labelled by its position on the curve (low latency, mid-curve, max throughput) and comes with its topology, GPU count and benchmark concurrencies, so you can deploy a specific point from 2 to 18 nodes and reproduce it with the InferenceX 8k/1k methodology. + +| Model | Points | Topologies | Recipe | +| :--- | :--- | :--- | :--- | +| **DeepSeek-V4-Pro (MXFP4 + MTP)** | p1 to p8 | 1P1D TP4 (8 GPUs) up to 8P1D DEP4/DEP8 (40 GPUs) | [pareto-sweep/README.md#points-dsv4](./pareto-sweep/README.md#points-dsv4) | +| **DeepSeek-R1 671B (NVFP4)** | low_latency, mid_curve, max_tpt | 1P4D TP4 (20 GPUs), 6P1D TP4/DEP48 and 10P1D TP4/DEP32 (72 GPUs) | [pareto-sweep/README.md#points-dsr1-fp4](./pareto-sweep/README.md#points-dsr1-fp4) | +| **DeepSeek-R1 671B (FP8)** | low-latency, mid, max | 1P1D TP4 (8 GPUs), 5P1D DEP8/DEP32 and 6P1D DEP8/DEP24 (72 GPUs) | [pareto-sweep/README.md#points-dsr1-fp8](./pareto-sweep/README.md#points-dsr1-fp8) | + ## 4. Inference Request and Benchmarking diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md new file mode 100644 index 00000000..b4b63a6b --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/README.md @@ -0,0 +1,571 @@ +# DeepSeek-V4-Pro and DeepSeek-R1 Pareto-Sweep Recipes with NVIDIA Dynamo on A4X-MAX GKE + +This recipe packages the operating points of the public [InferenceX](https://inferencex.semianalysis.com/inference) GB300 Pareto curves for **DeepSeek-V4-Pro (MXFP4 + EAGLE MTP)**, **DeepSeek-R1 (NVFP4)** and **DeepSeek-R1 (FP8)** as deployable [NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) + [SGLang](https://github.com/sgl-project/sglang) disaggregated deployments on [A4X-MAX (GB300 NVL72) GKE node pools](https://cloud.google.com/compute/docs/accelerator-optimized-machines#a4x-vms). + +Every point on a Pareto curve is a tuple of **topology** (how many prefill and decode workers, how many nodes each spans), **serving flags** (parallelism, MoE backend, memory, CUDA graphs, speculative decoding) and **benchmark concurrency**. This recipe turns each tuple into one Helm values file plus a prefill and a decode serving configuration, so you can deploy any point on an A4X-MAX NVL72 domain and reproduce the measurement with the same benchmark client and methodology. It builds on the generic [A4X-MAX disaggregated Dynamo recipe](../README.md) and reuses its Helm chart, platform install and benchmark client. + +> [!NOTE] +> This document describes **where each operating point sits on the Pareto curve** (low latency, mid-curve, max throughput) and how to deploy and measure it. It intentionally does not publish throughput or latency figures. + + +## Table of Contents + +* [1. Test Environment](#test-environment) +* [2. High-Level Architecture](#architecture) + * [2.1. What Gets Deployed](#what-gets-deployed) + * [2.2. Request Path](#request-path) + * [2.3. How a Point Fills the NVLink Domain](#how-a-point-fills-the-nvlink-domain) +* [3. Environment Setup (One-Time)](#environment-setup) + * [3.1. Shared Setup from the Parent Recipe](#shared-setup) + * [3.2. Stage Model Weights](#stage-weights) + * [3.3. Configure the Dynamo Image](#configure-image) +* [4. Operating Points and Pareto Maps](#operating-points) + * [4.1. How to Read the Tables](#how-to-read) + * [4.2. DeepSeek-V4-Pro (MXFP4 + EAGLE MTP)](#points-dsv4) + * [4.3. DeepSeek-R1 (NVFP4)](#points-dsr1-fp4) + * [4.4. DeepSeek-R1 (FP8)](#points-dsr1-fp8) +* [5. Deploy an Operating Point](#deploy) +* [6. Benchmark an Operating Point](#benchmark) +* [7. Monitoring and Troubleshooting](#monitoring) +* [8. Cleanup](#cleanup) +* [9. Sources](#sources) + + +## 1. Test Environment + +[Back to Top](#table-of-contents) + +This recipe has been tested with the following configuration: + +* **GKE Cluster**: + * A GPU node pool with [a4x-maxgpu-4g-metal](https://docs.cloud.google.com/compute/docs/gpus#gb300-gpus) machines (4 × GB300 per node) that belong to **one NVL72 NVLink domain**. Operating points need between 2 and 18 nodes (8 to 72 GPUs); the point tables below give the exact count. + * [Workload Identity Federation for GKE](https://cloud.google.com/kubernetes-engine/docs/concepts/workload-identity) enabled + * [Cloud Storage FUSE CSI driver for GKE](https://cloud.google.com/kubernetes-engine/docs/concepts/cloud-storage-fuse-csi-driver) enabled + * NVIDIA Dynamo Platform (operator, Grove and KAI scheduler) installed as described in the [parent recipe](../README.md#install-platform) +* **Software stack used for the measurements** (the serving flags in this recipe were measured with these versions; see [3.3](#configure-image) for how to build matching images): + +| Model | Weights | SGLang image | Dynamo | KV transfer | MoE / speculative decoding | +| :--- | :--- | :--- | :--- | :--- | :--- | +| DeepSeek-V4-Pro (MXFP4) | `deepseek-ai/DeepSeek-V4-Pro` | `lmsysorg/sglang:v0.5.13-cu130` | Dynamo 1.2.0 (`ai-dynamo==1.2.0.post1`) | Mooncake over NVLink (MNNVL) | DeepGEMM mega-MoE (`megamoe`), EAGLE MTP on decode | +| DeepSeek-R1 (NVFP4) | `nvidia/DeepSeek-R1-0528-NVFP4-v2`, served as `deepseek-ai/DeepSeek-R1` | `lmsysorg/sglang:v0.5.8.post1-cu130-runtime` | `ai-dynamo==0.8.1` | NIXL over NVLink (MNNVL) | DeepEP low-latency wide-EP on decode, no speculative decoding | +| DeepSeek-R1 (FP8) | `deepseek-ai/DeepSeek-R1-0528`, served as `deepseek-ai/DeepSeek-R1` | `lmsysorg/sglang:v0.5.8.post1-cu130` | `ai-dynamo==0.8.0` | NIXL over NVLink (MNNVL) | DeepEP normal (prefill) / low-latency (decode) wide-EP, no speculative decoding | + +The DeepSeek-V4-Pro points were measured on pre-release SGLang and Dynamo builds. SGLang v0.5.13 and Dynamo 1.2.0 are the first releases that contain those builds. + +All three stacks keep KV-cache transfer and MoE all-to-all traffic on the NVL72 fabric (`MC_FORCE_MNNVL=1`, `NCCL_MNNVL_ENABLE=1`, `UCX_TLS=cuda_copy,cuda_ipc,tcp`), so no RDMA NIC resource claims are requested. + + +## 2. High-Level Architecture + +[Back to Top](#table-of-contents) + +Every operating point has the same shape. A pool of **prefill workers** each owns a few GPUs. A **decode worker**, usually one wide expert-parallel (DEP) group, spans several nodes. **Dynamo frontends** split each request into a prefill leg and a decode leg. All worker pods sit inside one `ComputeDomain`, so prefill hands its KV cache to decode over NVLink (MNNVL) even across nodes. + +The three diagrams below show this from three angles: what gets deployed, how a request flows, and how one point fills the NVLink domain. + +### 2.1. What Gets Deployed + +```mermaid +flowchart LR + V["Values file
one per operating point"] --> H["helm install"] + C["Serving configs
prefill + decode"] --> H + H --> OP["Dynamo Operator"] + OP --> FE["Frontend pods"] + OP -->|"worker pods"| CD + W[("Model weights
GCS FUSE")] -.->|"mounted at /data/model"| CD + subgraph CD["ComputeDomain: one NVL72 domain"] + direction TB + PF["Prefill workers"] + DC["Decode worker"] + end +``` + +| Piece | File | +| :--- | :--- | +| Values file (topology, `ComputeDomain` size, worker env) | One per point, in a folder per model: [`deepseek-v4-pro/`](deepseek-v4-pro/), [`deepseek-r1-nvfp4/`](deepseek-r1-nvfp4/) and [`deepseek-r1-fp8/`](deepseek-r1-fp8/). For example [`dsv4_p3_mid_curve.yaml`](deepseek-v4-pro/dsv4_p3_mid_curve.yaml). The tables in [section 4](#operating-points) link every one. | +| Serving configs (SGLang flags per role) | [`src/frameworks/a4xmax/dynamo-configs/`](../../../../../src/frameworks/a4xmax/dynamo-configs/), for example [`dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml`](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml) | +| Helm chart | [`src/helm-charts/a4xmax/inference-templates/dynamo-deployment`](../../../../../src/helm-charts/a4xmax/inference-templates/dynamo-deployment/) | +| Serving image | [`src/docker/dynamo-sglang-deepseek/Dockerfile`](../../../../../src/docker/dynamo-sglang-deepseek/Dockerfile) | +| Benchmark client | [`bench_client.yaml`](../bench_client.yaml) | + +### 2.2. Request Path + +```mermaid +flowchart LR + B["bench-client
8k in / 1k out"] -->|"/v1/completions"| F["Frontend"] + F -->|"prefill leg"| P["Prefill worker"] + P ==>|"KV cache over NVLink
Mooncake or NIXL"| D["Decode worker"] + D -->|"streamed tokens"| F + F --> B +``` + +The frontend picks a prefill worker and a decode worker for each request. Prefill computes the prompt's KV cache and pushes it to decode over NVLink. Decode then generates the output tokens and streams them back through the frontend. + +### 2.3. How a Point Fills the NVLink Domain + +Each worker occupies whole GB300 nodes (4 GPUs per node). As an example, point **p5** of DeepSeek-V4-Pro runs two single-node DEP4 prefill workers and one DEP8 decode worker that spans two nodes: + +```mermaid +flowchart TB + subgraph CD["ComputeDomain for p5: 4 nodes / 16 GPUs"] + direction LR + subgraph PW2["Prefill 2 · DEP4"] + N2["node 2"] + end + subgraph PW1["Prefill 1 · DEP4"] + N1["node 1"] + end + subgraph DW["Decode · DEP8"] + direction LR + N3["node 3
leader"] --- N4["node 4"] + end + end +``` + +The values file encodes this layout with three knobs: + +| Recipe knob | Where it lives | What it controls | +| :--- | :--- | :--- | +| `dynamo.prefillWorker.replicas` / `nodeCount` | values file | Number of prefill workers and nodes per worker. TP4 and DEP4 workers use 1 node; DEP8 workers use 2. | +| `dynamo.decodeWorker.replicas` / `nodeCount` | values file | Number of decode workers and nodes per worker. DEP8 is 2 nodes, DEP16 is 4, DEP24 is 6, DEP32 is 8 and DEP48 is 12. | +| `dynamo.computeDomain.numNodes` | values file | Prefill replicas × nodeCount plus decode replicas × nodeCount. All pods of a point share this NVLink domain. | +| `*-prefill.yaml` / `*-decode.yaml` | [`src/frameworks/a4xmax/dynamo-configs/`](../../../../../src/frameworks/a4xmax/dynamo-configs/) | SGLang serving flags for each role, taken flag-for-flag from the InferenceX / srt-slurm recipe of that point. | +| worker `envs` | values file | Transport (MNNVL/UCX), DeepGEMM / DeepEP tuning and DeepSeek-V4 reasoning env vars from the same recipe. | + + +## 3. Environment Setup (One-Time) + +[Back to Top](#table-of-contents) + + +### 3.1. Shared Setup from the Parent Recipe + +Complete sections [2.1 to 2.6 of the parent recipe](../README.md#environment-setup) first (clone the repository, configure variables, connect to the cluster, create secrets, install the Dynamo Platform, set up the GCS bucket). Then point `RECIPE_ROOT` at this directory: + +```bash +export RECIPE_ROOT=$REPO_ROOT/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep +``` + + +### 3.2. Stage Model Weights + +Stage the checkpoint of the model you want to sweep into `gs://${GCS_BUCKET}/` so that the chart mounts it at `/data/model/`: + +```bash +pip install -U "huggingface_hub[cli]" + +# DeepSeek-V4-Pro (MXFP4) +hf download deepseek-ai/DeepSeek-V4-Pro --local-dir ./deepseek-ai/DeepSeek-V4-Pro +gcloud storage cp -r ./deepseek-ai gs://${GCS_BUCKET}/ + +# DeepSeek-R1 NVFP4 (about 413 GB) +hf download nvidia/DeepSeek-R1-0528-NVFP4-v2 --local-dir ./nvidia/DeepSeek-R1-0528-NVFP4-v2 +gcloud storage cp -r ./nvidia gs://${GCS_BUCKET}/ + +# DeepSeek-R1 FP8 (about 689 GB) +hf download deepseek-ai/DeepSeek-R1-0528 --local-dir ./deepseek-ai/DeepSeek-R1-0528 +gcloud storage cp -r ./deepseek-ai gs://${GCS_BUCKET}/ +``` + +> [!TIP] +> The 72-GPU points load the checkpoint on 18 nodes at once. The chart enables the GCS FUSE file cache on local SSD, so the first start of a point is the slowest; later points on the same nodes start much faster. + + +### 3.3. Configure the Dynamo Image + +Each model family needs its own image, built to the SGLang + Dynamo versions in the table in [section 1](#test-environment). All three use the same [`Dockerfile`](../../../../../src/docker/dynamo-sglang-deepseek/Dockerfile) in [`src/docker/dynamo-sglang-deepseek/`](../../../../../src/docker/dynamo-sglang-deepseek/) with different build arguments. Build the images for the models you plan to sweep and push them to Artifact Registry: + +```bash +export REGION= +export AR_REPO= +cd $REPO_ROOT/src/docker/dynamo-sglang-deepseek + +# DeepSeek-R1 NVFP4 (SGLang v0.5.8.post1 + ai-dynamo 0.8.1) +docker buildx build --platform linux/arm64 \ + --build-arg BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130-runtime --build-arg DYNAMO_VERSION=0.8.1 \ + -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsr1:0.8.1 --push . + +# DeepSeek-R1 FP8 (SGLang v0.5.8.post1 + ai-dynamo 0.8.0) +docker buildx build --platform linux/arm64 \ + --build-arg BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130 --build-arg DYNAMO_VERSION=0.8.0 \ + -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsr1:0.8.0 --push . + +# DeepSeek-V4-Pro (SGLang v0.5.13 + Dynamo 1.2.0) +docker buildx build --platform linux/arm64 \ + --build-arg BASE_IMAGE=lmsysorg/sglang:v0.5.13-cu130 --build-arg DYNAMO_VERSION=1.2.0.post1 \ + -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsv4:1.2.0.post1 --push . +``` + +Then point `DYNAMO_IMAGE` at the image for the model of the point you deploy: + +| Model | `DYNAMO_IMAGE` | +| :--- | :--- | +| DeepSeek-V4-Pro | `${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsv4:1.2.0.post1` | +| DeepSeek-R1 (NVFP4) | `${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsr1:0.8.1` | +| DeepSeek-R1 (FP8) | `${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsr1:0.8.0` | + +```bash +# Example: DeepSeek-V4-Pro, used by the point deployed in section 5 +export DYNAMO_IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/dynamo-sglang-dsv4:1.2.0.post1" +``` + +> [!WARNING] +> Do not use the stock `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.0.1-cuda13` image for this recipe. It ships SGLang v0.5.9, which does not accept `--moe-a2a-backend megamoe`, so DeepSeek-V4-Pro points p2 to p8 fail during argument parsing. The DeepSeek-R1 points were not validated on it either. + +The install command in [section 5](#deploy) also passes `--set dynamo.frontend.image=${DYNAMO_IMAGE}`, so the frontend runs the same Dynamo build as the workers. + + +## 4. Operating Points and Pareto Maps + +[Back to Top](#table-of-contents) + + +### 4.1. How to Read the Tables + +Each Pareto curve plots **total token throughput per GPU** against **interactivity** (tokens per second per user, the inverse of time per output token). Moving along the curve from left to right trades interactivity for throughput: + +* **Low latency**: small, TP-only or few-node topologies at very low concurrency. Highest interactivity, lowest throughput per GPU. +* **Mid-curve**: wide expert-parallel decode groups at moderate concurrency. Balanced. +* **Max throughput**: the largest prefill pools and the highest concurrency the decode group can hold. Highest throughput per GPU, lowest interactivity. + +Column legend for the tables below: + +| Column | Meaning | +| :--- | :--- | +| **Pareto position** | Where the point sits on the curve (see above) | +| **Prefill / Decode** | `workers × parallelism (nodes per worker)`. `TPn` is tensor parallel; `DEPn` is data-parallel attention + expert parallel over `n` GPUs | +| **Nodes / GPUs** | Size of the `ComputeDomain` the point needs | +| **Concurrency** | The benchmark concurrencies swept against this topology (one deployment, several measurements) | +| **Values file** | Helm values in this directory (topology + env) | +| **Serving configs** | Prefix of the `-prefill.yaml` / `-decode.yaml` pair in [`src/frameworks/a4xmax/dynamo-configs/`](../../../../../src/frameworks/a4xmax/dynamo-configs/) | +| **GKE status** | Whether the point has been measured on A4X-MAX GKE with this recipe's flags | + + +### 4.2. DeepSeek-V4-Pro (MXFP4 + EAGLE MTP) + +Source: InferenceX `dsv4-fp4-gb300-dynamo-sglang-mtp`, srt-slurm `recipes/dsv4-pro/sglang/gb300-fp4/8k1k/disagg/`. All decode workers run EAGLE multi-token prediction; the mid-curve and max-throughput points also use DeepGEMM mega-MoE with data-parallel attention on both roles. + +```mermaid +--- +config: + layout: dagre +--- +flowchart LR + classDef ll fill:#E3F2FD,stroke:#1565C0,color:#0D47A1 + classDef mid fill:#E8F5E9,stroke:#2E7D32,color:#1B5E20 + classDef max fill:#FFF3E0,stroke:#EF6C00,color:#E65100 + classDef axis fill:none,stroke:none,color:#555 + + AX1["◄ higher interactivity (tok/s/user)"]:::axis + AX2["higher throughput per GPU ►"]:::axis + + subgraph LL["Low latency"] + direction TB + p1["p1 · 1P1D
TP4 / TP4 · 8 GPUs
concurrency 1"]:::ll + p2["p2 · 1P6D
DEP4 / 6 × TP4 · 28 GPUs
concurrency 8 · 32 · 64"]:::ll + end + subgraph MID["Mid-curve"] + direction TB + p3["p3 · 1P1D
DEP4 / DEP8 · 12 GPUs
concurrency 256"]:::mid + p4["p4 · 1P1D
DEP4 / DEP16 · 20 GPUs
concurrency 256"]:::mid + p5["p5 · 2P1D
2 × DEP4 / DEP8 · 16 GPUs
concurrency 512"]:::mid + p6["p6 · 4P1D
4 × DEP4 / DEP8 · 24 GPUs
concurrency 1024"]:::mid + end + subgraph MAX["Max throughput"] + direction TB + p7["p7 · 6P1D
6 × DEP4 / DEP8 · 32 GPUs
concurrency 4096"]:::max + p8["p8 · 8P1D
8 × DEP4 / DEP8 · 40 GPUs
concurrency 8192"]:::max + end + + AX1 ~~~ LL + p1 --> p2 --> p3 --> p4 --> p5 --> p6 --> p7 --> p8 + MAX ~~~ AX2 +``` + +| Point | Pareto position | Prefill | Decode | Nodes / GPUs | Concurrency | Values file | Serving configs | GKE status | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| **p1** | Low latency | 1 × TP4 (1 node) | 1 × TP4 (1 node) | 2 / 8 | 1 | [`dsv4_p1_low_latency.yaml`](deepseek-v4-pro/dsv4_p1_low_latency.yaml) | `dsv4-p1-low-latency-1p1d-tp4-tp4` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml)) | Measured | +| **p2** | Low latency | 1 × DEP4 (1 node) | 6 × TP4 (1 node each) | 7 / 28 | 8, 32, 64 | [`dsv4_p2_low_latency.yaml`](deepseek-v4-pro/dsv4_p2_low_latency.yaml) | `dsv4-p2-low-latency-1p6d-dep4-tp4` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml)) | Measured | +| **p3** | Mid-curve | 1 × DEP4 (1 node) | 1 × DEP8 (2 nodes) | 3 / 12 | 256 | [`dsv4_p3_mid_curve.yaml`](deepseek-v4-pro/dsv4_p3_mid_curve.yaml) | `dsv4-p3-mid-curve-1p1d-dep4-dep8` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml)) | Measured | +| **p4** | Mid-curve | 1 × DEP4 (1 node) | 1 × DEP16 (4 nodes) | 5 / 20 | 256 | [`dsv4_p4_mid_curve.yaml`](deepseek-v4-pro/dsv4_p4_mid_curve.yaml) | `dsv4-p4-mid-curve-1p1d-dep4-dep16` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml)) | Measured | +| **p5** | Mid-curve | 2 × DEP4 (1 node each) | 1 × DEP8 (2 nodes) | 4 / 16 | 512 | [`dsv4_p5_mid_curve.yaml`](deepseek-v4-pro/dsv4_p5_mid_curve.yaml) | `dsv4-p5-mid-curve-2p1d-dep4-dep8` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml)) | Measured | +| **p6** | Mid-curve | 4 × DEP4 (1 node each) | 1 × DEP8 (2 nodes) | 6 / 24 | 1024 | [`dsv4_p6_mid_curve.yaml`](deepseek-v4-pro/dsv4_p6_mid_curve.yaml) | `dsv4-p6-mid-curve-4p1d-dep4-dep8` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml)) | Measured | +| **p7** | Max throughput | 6 × DEP4 (1 node each) | 1 × DEP8 (2 nodes) | 8 / 32 | 4096 | [`dsv4_p7_max_throughput.yaml`](deepseek-v4-pro/dsv4_p7_max_throughput.yaml) | `dsv4-p7-max-throughput-6p1d-dep4-dep8` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml)) | Measured | +| **p8** | Max throughput (end of curve) | 8 × DEP4 (1 node each) | 1 × DEP8 (2 nodes) | 10 / 40 | 8192 | [`dsv4_p8_max_throughput.yaml`](deepseek-v4-pro/dsv4_p8_max_throughput.yaml) | `dsv4-p8-max-throughput-8p1d-dep4-dep8` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml)) | Measured (latency-only rerun) | + +What changes along the curve: the decode worker keeps the same DEP8 shape from p3 onward while the prefill pool grows from 1 to 8 workers; the decode side trades speculative depth for batch capacity (EAGLE steps 3 → 2 → 1, larger `max-running-requests` and CUDA-graph batch, shorter context window) as concurrency rises. + + +### 4.3. DeepSeek-R1 (NVFP4) + +Source: InferenceX `dsr1-fp4-gb300-dynamo-sglang`, srt-slurm `recipes/gb300-fp4/8k1k/{low_latency,mid_curve,max_tpt}.yaml`. Prefill workers are always single-node TP4; the mid-curve and max-throughput points use one wide DeepEP low-latency decode group that spans 12 or 8 nodes. + +```mermaid +--- +config: + layout: dagre +--- +flowchart LR + classDef ll fill:#E3F2FD,stroke:#1565C0,color:#0D47A1 + classDef mid fill:#E8F5E9,stroke:#2E7D32,color:#1B5E20 + classDef max fill:#FFF3E0,stroke:#EF6C00,color:#E65100 + classDef axis fill:none,stroke:none,color:#555 + + AX1["◄ higher interactivity (tok/s/user)"]:::axis + AX2["higher throughput per GPU ►"]:::axis + + subgraph LL["Low latency"] + ll["low_latency · 1P4D
TP4 / 4 × TP4 · 20 GPUs
concurrency 4 · 8 · 32 · 64"]:::ll + end + subgraph MID["Mid-curve"] + mid["mid_curve · 6P1D
6 × TP4 / DEP48 · 72 GPUs
concurrency 512 · 2048 · 4096"]:::mid + end + subgraph MAX["Max throughput"] + max["max_tpt · 10P1D
10 × TP4 / DEP32 · 72 GPUs
concurrency 2048"]:::max + end + + AX1 ~~~ LL + ll --> mid --> max + MAX ~~~ AX2 +``` + +| Point | Pareto position | Prefill | Decode | Nodes / GPUs | Concurrency | Values file | Serving configs | GKE status | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| **low_latency** | Low latency | 1 × TP4 (1 node) | 4 × TP4 (1 node each) | 5 / 20 | 4, 8, 32, 64 | [`dsr1_fp4_low_latency.yaml`](deepseek-r1-nvfp4/dsr1_fp4_low_latency.yaml) | `dsr1-fp4-low-latency-1p4d-tp4-tp4` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml)) | Measured | +| **mid_curve** | Mid-curve | 6 × TP4 (1 node each) | 1 × DEP48 (12 nodes) | 18 / 72 | 512, 2048, 4096 | [`dsr1_fp4_mid_curve.yaml`](deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml) | `dsr1-fp4-mid-curve-6p1d-tp4-dep48` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml)) | Measured | +| **max_tpt** | Max throughput | 10 × TP4 (1 node each) | 1 × DEP32 (8 nodes) | 18 / 72 | 2048 | [`dsr1_fp4_max_throughput.yaml`](deepseek-r1-nvfp4/dsr1_fp4_max_throughput.yaml) | `dsr1-fp4-max-throughput-10p1d-tp4-dep32` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml)) | Measured | + +What changes along the curve: the whole NVL72 rack is used for both wide points, but the max-throughput point moves 4 nodes from decode to prefill (10 prefill / 8 decode nodes instead of 6 / 12) so that prefill keeps the decode group saturated at high concurrency. + + +### 4.4. DeepSeek-R1 (FP8) + +Source: InferenceX `dsr1-fp8-gb300-dynamo-sglang`, srt-slurm `recipes/gb300-fp8/8k1k/stp/{low-latency,mid,max}.yaml`. Unlike the NVFP4 curve, the wide points run **multi-node prefill workers** (DEP8 = 2 nodes per worker, DeepEP normal mode) in front of a DEP32 or DEP24 decode group. + +```mermaid +--- +config: + layout: dagre +--- +flowchart LR + classDef ll fill:#E3F2FD,stroke:#1565C0,color:#0D47A1 + classDef mid fill:#E8F5E9,stroke:#2E7D32,color:#1B5E20 + classDef max fill:#FFF3E0,stroke:#EF6C00,color:#E65100 + classDef axis fill:none,stroke:none,color:#555 + + AX1["◄ higher interactivity (tok/s/user)"]:::axis + AX2["higher throughput per GPU ►"]:::axis + + subgraph LL["Low latency"] + ll["low-latency · 1P1D
TP4 / TP4 · 8 GPUs
concurrency 4 · 8"]:::ll + end + subgraph MID["Mid-curve"] + mid["mid · 5P1D
5 × DEP8 / DEP32 · 72 GPUs
concurrency 128 · 256 · 512 · 1024"]:::mid + end + subgraph MAX["Max throughput"] + max["max · 6P1D
6 × DEP8 / DEP24 · 72 GPUs
concurrency 2048 · 4096"]:::max + end + + AX1 ~~~ LL + ll --> mid --> max + MAX ~~~ AX2 +``` + +| Point | Pareto position | Prefill | Decode | Nodes / GPUs | Concurrency | Values file | Serving configs | GKE status | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| **low-latency** | Low latency | 1 × TP4 (1 node) | 1 × TP4 (1 node) | 2 / 8 | 4, 8 | [`dsr1_fp8_low_latency.yaml`](deepseek-r1-fp8/dsr1_fp8_low_latency.yaml) | `dsr1-fp8-low-latency-1p1d-tp4-tp4` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml)) | Measured at concurrency 4 | +| **mid** | Mid-curve | 5 × DEP8 (2 nodes each) | 1 × DEP32 (8 nodes) | 18 / 72 | 128, 256, 512, 1024 | [`dsr1_fp8_mid_curve.yaml`](deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml) | `dsr1-fp8-mid-curve-5p1d-dep8-dep32` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml)) | Measured | +| **max** | Max throughput | 6 × DEP8 (2 nodes each) | 1 × DEP24 (6 nodes) | 18 / 72 | 2048, 4096 | [`dsr1_fp8_max_throughput.yaml`](deepseek-r1-fp8/dsr1_fp8_max_throughput.yaml) | `dsr1-fp8-max-throughput-6p1d-dep8-dep24` ([prefill](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml), [decode](../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml)) | Configured from the InferenceX recipe; not yet measured on GKE | + +> [!NOTE] +> The parent recipe's generic [`values_wo_deepep.yaml`](../values_wo_deepep.yaml) (1P1D) and [`values_deepep.yaml`](../values_deepep.yaml) (10P8D) DeepSeek-R1 FP8 deployments use the same two topologies as the **low-latency** and **mid** points here but with the current Dynamo runtime's serving flags. Use this directory when you want the InferenceX point definitions verbatim. + + +## 5. Deploy an Operating Point + +[Back to Top](#table-of-contents) + +Pick a row from the tables in [section 4](#operating-points) and set the four variables below from its **Values file** and **Serving configs** columns. `POINT_VALUES` is the path relative to this directory, including the model folder. The example deploys DeepSeek-V4-Pro point **p3** (mid-curve, 3 nodes). + +1. Set the point and model variables: + +```bash +export POINT_VALUES="deepseek-v4-pro/dsv4_p3_mid_curve.yaml" +export POINT_CONFIG="dsv4-p3-mid-curve-1p1d-dep4-dep8" +export HF_MODEL_ID="deepseek-ai/DeepSeek-V4-Pro" # nvidia/DeepSeek-R1-0528-NVFP4-v2 or deepseek-ai/DeepSeek-R1-0528 for DeepSeek-R1 +export RELEASE_NAME="$USER-dsv4-p3" +export DEPLOYMENT_NAME="dsv4-p3" # dynamo.deploymentName in the values file +``` + +The Dynamo operator names the frontend, prefill and decode resources after `DEPLOYMENT_NAME` (for example `dsv4-p3-frontend`), not after the Helm release name. + +2. Install the Helm release: + +```bash +cd $RECIPE_ROOT +helm install -f ${POINT_VALUES} \ + --set workload.model=${HF_MODEL_ID} \ + --set workload.image=${DYNAMO_IMAGE} \ + --set dynamo.frontend.image=${DYNAMO_IMAGE} \ + --set volumes.gcsfuse.bucketName=${GCS_BUCKET} \ + --set dynamo.namespace=${NAMESPACE} \ + --set dynamo.nodeSelector."cloud\.google\.com/gke-nodepool"=${NODE_POOL} \ + --set-file prefill_serving_config=$REPO_ROOT/src/frameworks/a4xmax/dynamo-configs/${POINT_CONFIG}-prefill.yaml \ + --set-file decode_serving_config=$REPO_ROOT/src/frameworks/a4xmax/dynamo-configs/${POINT_CONFIG}-decode.yaml \ + ${RELEASE_NAME} \ + $REPO_ROOT/src/helm-charts/a4xmax/inference-templates/dynamo-deployment +``` + +Keep the `--set dynamo.frontend.image=...` line: the values files default the frontend to `dynamo-frontend:1.0.1`, which does not match the Dynamo build of the workers. + +3. Wait for every pod of the point to become `Running` and `Ready`: + +```bash +kubectl get pods -n ${NAMESPACE} -w +``` + +A multi-node decode group appears as one leader pod (`...-decode-0-decode-ldr-...`) plus `nodeCount - 1` worker pods; multi-node prefill workers (FP8 **mid** / **max**) appear the same way for each prefill replica. Model load for the 72-GPU points can take tens of minutes on a cold GCS FUSE cache. + +> [!IMPORTANT] +> Deploy **one point at a time per NVL72 domain** and uninstall it before the next one. Each values file sizes its own `ComputeDomain` and the 72-GPU points need the whole domain. + + +## 6. Benchmark an Operating Point + +[Back to Top](#table-of-contents) + +All points are measured with the InferenceX methodology: the vLLM-style `benchmark_serving.py` client ("sa-bench"), a **random dataset with 8192 input / 1024 output tokens** (`--random-range-ratio 0.8`, `--ignore-eos`), unlimited request rate, and the concurrencies listed for the point. Each concurrency is run three times against the standing deployment: a short warm-up, a longer discarded warm run, then the measured run. + +```mermaid +flowchart LR + A["Deploy point
(section 5)"] --> B["Verify readiness
frontend serves /v1/models"] + B --> C["Warm-up
3 × concurrency prompts
request rate 250"] + C --> D["Discarded warm run
5 × concurrency prompts
request rate inf"] + D --> E["Measured run
10 × concurrency prompts
request rate inf"] + E -->|next concurrency| C + E --> F["Collect results JSON
normalize per role"] +``` + +Throughput is normalized per role: **input tokens/s ÷ prefill GPUs**, **output tokens/s ÷ decode GPUs**, and **total tokens/s ÷ all GPUs** of the point. Interactivity is `1000 / mean TPOT (ms)`. + +1. Deploy the benchmark client from the parent recipe and wait for `--- SETUP COMPLETE. POD IS READY. ---`: + +```bash +kubectl apply -f $RECIPE_ROOT/../bench_client.yaml -n ${NAMESPACE} +kubectl logs -f pod/bench-client -n ${NAMESPACE} +``` + +2. Set the frontend, model names and the concurrencies of the point (from the **Concurrency** column): + +```bash +export FRONTEND_HOST="$(kubectl get svc -n ${NAMESPACE} -o name | grep "/${DEPLOYMENT_NAME}-frontend" | head -n1 | cut -d/ -f2)" +[ -n "${FRONTEND_HOST}" ] || { echo "No frontend service for ${DEPLOYMENT_NAME} in ${NAMESPACE}"; false; } +export SERVED_MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" # deepseek-ai/DeepSeek-R1 for both DeepSeek-R1 variants +export TOKENIZER="${HF_MODEL_ID}" +export CONCURRENCIES="256" # e.g. "4 8 32 64" for a low-latency point +``` + +3. Run the sweep from the `bench-client` pod. DeepSeek-R1 uses the client bundled in the Dynamo repository checkout: + +```bash +kubectl exec -it bench-client -n ${NAMESPACE} -- bash -c " + cd /workspace/dynamo/examples/backends/sglang/slurm_jobs/scripts/vllm && mkdir -p /logs/results + run_bench() { # concurrency num_prompts request_rate save_flags + python3 -u benchmark_serving.py \ + --model ${SERVED_MODEL_NAME} --tokenizer ${TOKENIZER} \ + --host ${FRONTEND_HOST} --port 8000 \ + --backend dynamo --endpoint /v1/completions \ + --disable-tqdm --dataset-name random \ + --num-prompts \$2 \ + --random-input-len 8192 --random-output-len 1024 --random-range-ratio 0.8 \ + --ignore-eos --request-rate \$3 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --max-concurrency \$1 --trust-remote-code \$4 + } + for C in ${CONCURRENCIES}; do + run_bench \$C \$((C*3)) 250 '' + run_bench \$C \$((C*5)) inf '' + run_bench \$C \$((C*10)) inf \"--save-result --result-dir /logs/results --result-filename results_concurrency_\${C}.json\" + done +" +``` + +DeepSeek-V4-Pro must be benchmarked through its chat template with the DeepSeek-V4 tokenizer wrapper that ships with the srt-slurm copy of sa-bench. Clone it once inside the pod and add the two extra flags: + +```bash +kubectl exec -it bench-client -n ${NAMESPACE} -- bash -c " + [ -d /workspace/srt-slurm ] || git clone --depth 1 https://github.com/NVIDIA/srt-slurm.git /workspace/srt-slurm + cd /workspace/srt-slurm/src/srtctl/benchmarks/scripts/sa-bench && mkdir -p /logs/results + run_bench() { + python3 -u benchmark_serving.py \ + --model ${SERVED_MODEL_NAME} --tokenizer ${TOKENIZER} \ + --host ${FRONTEND_HOST} --port 8000 \ + --backend dynamo --endpoint /v1/completions \ + --disable-tqdm --dataset-name random \ + --num-prompts \$2 \ + --random-input-len 8192 --random-output-len 1024 --random-range-ratio 0.8 \ + --ignore-eos --request-rate \$3 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --max-concurrency \$1 --trust-remote-code \ + --use-chat-template \ + --custom-tokenizer sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer \$4 + } + for C in ${CONCURRENCIES}; do + run_bench \$C \$((C*3)) 250 '' + run_bench \$C \$((C*5)) inf '' + run_bench \$C \$((C*10)) inf \"--save-result --result-dir /logs/results --result-filename results_concurrency_\${C}.json\" + done +" +``` + +4. Copy the result files out of the pod when the sweep finishes: + +```bash +kubectl cp ${NAMESPACE}/bench-client:/logs/results ./results-${RELEASE_NAME} +``` + +Each JSON file contains `total_token_throughput`, `output_throughput`, the TTFT / TPOT / ITL / E2E percentiles and the request count; divide the throughputs by the GPU counts in the point table to place the point on the curve. + + +## 7. Monitoring and Troubleshooting + +[Back to Top](#table-of-contents) + +Find pod names and follow logs as in the [parent recipe](../README.md#monitoring): + +```bash +kubectl get pods -n ${NAMESPACE} | grep ${DEPLOYMENT_NAME} +kubectl logs -f -c main -n ${NAMESPACE} +``` + +Checks that matter for Pareto reproduction: + +* **KV transfer is on NVLink.** Worker logs should show Mooncake / NIXL using `cuda_ipc` transports and `MC_FORCE_MNNVL=1`; if you see TCP fallbacks the pods are not in the same `ComputeDomain` or the node pool spans more than one NVL72 domain. +* **JIT kernels compile before you measure.** DeepGEMM and FlashInfer compile kernels on first use; the warm-up and discarded runs in [section 6](#benchmark) exist to push those compiles out of the measured window. If the measured run shows a TTFT tail far above the median, repeat the measured run. +* **Decode group does not form.** A multi-node decode worker waits for all `nodeCount` pods; a `Pending` worker pod usually means the node pool has fewer free nodes than `computeDomain.numNodes`. +* **Multi-node prefill (FP8 mid / max).** Each DEP8 prefill replica needs two nodes; five or six replicas plus the decode group need the whole 18-node domain. +* **Frontend readiness.** The frontend only reports healthy after at least one prefill and one decode worker have registered; `curl http://:8000/v1/models` lists the served model name once routing is possible. +* **DeepEP timeouts** on the wide FP8 points: see the DeepEP note in the [parent recipe](../README.md#monitoring). + + +## 8. Cleanup + +[Back to Top](#table-of-contents) + +Uninstall the point and the benchmark client: + +```bash +helm uninstall ${RELEASE_NAME} -n ${NAMESPACE} +kubectl delete -f $RECIPE_ROOT/../bench_client.yaml -n ${NAMESPACE} --ignore-not-found=true +``` + +To remove the Dynamo platform, namespace and secrets, follow the [parent recipe's cleanup](../README.md#cleanup). + + +## 9. Sources + +[Back to Top](#table-of-contents) + +* InferenceX GB300 Dynamo + SGLang configurations: [`SemiAnalysisAI/InferenceX`](https://github.com/SemiAnalysisAI/InferenceX) (`dsv4-fp4-gb300-dynamo-sglang-mtp`, `dsr1-fp4-gb300-dynamo-sglang`, `dsr1-fp8-gb300-dynamo-sglang`) and the public [dashboard](https://inferencex.semianalysis.com/inference). +* Per-point serving recipes: [`NVIDIA/srt-slurm`](https://github.com/NVIDIA/srt-slurm) (`recipes/dsv4-pro/sglang/gb300-fp4/8k1k/disagg/`, `recipes/gb300-fp4/8k1k/`, `recipes/gb300-fp8/8k1k/stp/`) including the `deepep_config.json` embedded in the FP8 configs. +* GKE sweep harness, manifests and methodology notes: [`alisachen-google/gcp-dynamo-cuj/dynamo-disagg-sweep`](https://github.com/alisachen-google/gcp-dynamo-cuj/tree/main/dynamo-disagg-sweep). diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/README.md b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/README.md new file mode 100644 index 00000000..4fb29c9c --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/README.md @@ -0,0 +1,11 @@ +# DeepSeek-R1 (FP8): Values Files + +Helm values files for the DeepSeek-R1 (FP8) operating points on A4X-MAX. Each file sets the topology and worker environment of one point on the Pareto curve. + +Deployment steps, point details and the Pareto map are in the [pareto-sweep recipe](../README.md#points-dsr1-fp8). + +| Point | Pareto position | Values file | Serving configs | +| :--- | :--- | :--- | :--- | +| **low-latency** | Low latency | [`dsr1_fp8_low_latency.yaml`](dsr1_fp8_low_latency.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml) | +| **mid** | Mid-curve | [`dsr1_fp8_mid_curve.yaml`](dsr1_fp8_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml) | +| **max** | Max throughput | [`dsr1_fp8_max_throughput.yaml`](dsr1_fp8_max_throughput.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml) | diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_low_latency.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_low_latency.yaml new file mode 100644 index 00000000..2ca5941b --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_low_latency.yaml @@ -0,0 +1,221 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 — operating point `ll` (Low latency) +# Topology: 1 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 1 node(s) +# = 2 nodes / 8 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp8/8k1k/stp/low-latency` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp8-ll + computeDomain: + name: dsr1fp8-ll-domain + numNodes: 2 + resourceClaimTemplateName: dsr1fp8-ll-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "true" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "true" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-R1-0528 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp8" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_max_throughput.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_max_throughput.yaml new file mode 100644 index 00000000..d49d582d --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_max_throughput.yaml @@ -0,0 +1,221 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 — operating point `max` (Max throughput) +# Topology: 6 prefill worker(s) x 2 node(s) + 1 decode worker(s) x 6 node(s) +# = 18 nodes / 72 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp8/8k1k/stp/max` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp8-max + computeDomain: + name: dsr1fp8-max-domain + numNodes: 18 + resourceClaimTemplateName: dsr1fp8-max-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 6 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK + value: "768" + - name: SGLANG_HACK_SEQ_BOOTSTRAP_ROOM + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 2 + replicas: 6 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-R1-0528 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp8" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml new file mode 100644 index 00000000..841c4a3e --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-fp8/dsr1_fp8_mid_curve.yaml @@ -0,0 +1,221 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 — operating point `mid` (Mid-curve) +# Topology: 5 prefill worker(s) x 2 node(s) + 1 decode worker(s) x 8 node(s) +# = 18 nodes / 72 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp8/8k1k/stp/mid` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp8-mid + computeDomain: + name: dsr1fp8-mid-domain + numNodes: 18 + resourceClaimTemplateName: dsr1fp8-mid-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 8 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK + value: "768" + - name: SGLANG_HACK_SEQ_BOOTSTRAP_ROOM + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 2 + replicas: 5 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-R1-0528 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp8" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/README.md b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/README.md new file mode 100644 index 00000000..2d947796 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/README.md @@ -0,0 +1,11 @@ +# DeepSeek-R1 (NVFP4): Values Files + +Helm values files for the DeepSeek-R1 (NVFP4) operating points on A4X-MAX. Each file sets the topology and worker environment of one point on the Pareto curve. + +Deployment steps, point details and the Pareto map are in the [pareto-sweep recipe](../README.md#points-dsr1-fp4). + +| Point | Pareto position | Values file | Serving configs | +| :--- | :--- | :--- | :--- | +| **low_latency** | Low latency | [`dsr1_fp4_low_latency.yaml`](dsr1_fp4_low_latency.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml) | +| **mid_curve** | Mid-curve | [`dsr1_fp4_mid_curve.yaml`](dsr1_fp4_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml) | +| **max_tpt** | Max throughput | [`dsr1_fp4_max_throughput.yaml`](dsr1_fp4_max_throughput.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml) | diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_low_latency.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_low_latency.yaml new file mode 100644 index 00000000..ecee417a --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_low_latency.yaml @@ -0,0 +1,221 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 — operating point `ll` (Low latency) +# Topology: 1 prefill worker(s) x 1 node(s) + 4 decode worker(s) x 1 node(s) +# = 5 nodes / 20 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp4/8k1k/low_latency` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp4-ll + computeDomain: + name: dsr1fp4-ll-domain + numNodes: 5 + resourceClaimTemplateName: dsr1fp4-ll-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 1 + replicas: 4 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "false" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "false" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_max_throughput.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_max_throughput.yaml new file mode 100644 index 00000000..0abe150e --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_max_throughput.yaml @@ -0,0 +1,229 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 — operating point `max` (Max throughput) +# Topology: 10 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 8 node(s) +# = 18 nodes / 72 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp4/8k1k/max_tpt` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp4-max + computeDomain: + name: dsr1fp4-max-domain + numNodes: 18 + resourceClaimTemplateName: dsr1fp4-max-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 8 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN + value: "1" + - name: SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 + value: "1" + - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK + value: "512" + - name: SGLANG_MOE_NVFP4_DISPATCH + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 10 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN + value: "1" + - name: SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 + value: "1" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml new file mode 100644 index 00000000..2e5be960 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-r1-nvfp4/dsr1_fp4_mid_curve.yaml @@ -0,0 +1,229 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 — operating point `mid` (Mid-curve) +# Topology: 6 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 12 node(s) +# = 18 nodes / 72 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `gb300-fp4/8k1k/mid_curve` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsr1fp4-mid + computeDomain: + name: dsr1fp4-mid-domain + numNodes: 18 + resourceClaimTemplateName: dsr1fp4-mid-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 10 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 12 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_DECODE_BOOTSTRAP_TIMEOUT + value: "1000" + - name: SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN + value: "1" + - name: SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 + value: "1" + - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK + value: "512" + - name: SGLANG_MOE_NVFP4_DISPATCH + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 6 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT + value: "1800" + - name: DYN_SKIP_SGLANG_LOG_FORMATTING + value: "1" + - name: SGLANG_USE_MESSAGE_QUEUE_BROADCASTER + value: "0" + - name: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK + value: "1" + - name: SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN + value: "1" + - name: SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 + value: "1" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/README.md b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/README.md new file mode 100644 index 00000000..e13f2a97 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/README.md @@ -0,0 +1,16 @@ +# DeepSeek-V4-Pro (MXFP4 + EAGLE MTP): Values Files + +Helm values files for the DeepSeek-V4-Pro (MXFP4 + EAGLE MTP) operating points on A4X-MAX. Each file sets the topology and worker environment of one point on the Pareto curve. + +Deployment steps, point details and the Pareto map are in the [pareto-sweep recipe](../README.md#points-dsv4). + +| Point | Pareto position | Values file | Serving configs | +| :--- | :--- | :--- | :--- | +| **p1** | Low latency | [`dsv4_p1_low_latency.yaml`](dsv4_p1_low_latency.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml) | +| **p2** | Low latency | [`dsv4_p2_low_latency.yaml`](dsv4_p2_low_latency.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml) | +| **p3** | Mid-curve | [`dsv4_p3_mid_curve.yaml`](dsv4_p3_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml) | +| **p4** | Mid-curve | [`dsv4_p4_mid_curve.yaml`](dsv4_p4_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml) | +| **p5** | Mid-curve | [`dsv4_p5_mid_curve.yaml`](dsv4_p5_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml) | +| **p6** | Mid-curve | [`dsv4_p6_mid_curve.yaml`](dsv4_p6_mid_curve.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml) | +| **p7** | Max throughput | [`dsv4_p7_max_throughput.yaml`](dsv4_p7_max_throughput.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml) | +| **p8** | Max throughput (end of curve) | [`dsv4_p8_max_throughput.yaml`](dsv4_p8_max_throughput.yaml) | [prefill](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml), [decode](../../../../../../src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml) | diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p1_low_latency.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p1_low_latency.yaml new file mode 100644 index 00000000..7ce03c47 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p1_low_latency.yaml @@ -0,0 +1,223 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p1` (Low latency) +# Topology: 1 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 1 node(s) +# = 2 nodes / 8 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-low-latency-1p1d-tp4-tp4-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p1 + computeDomain: + name: dsv4-p1-domain + numNodes: 2 + resourceClaimTemplateName: dsv4-p1-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p2_low_latency.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p2_low_latency.yaml new file mode 100644 index 00000000..b44e3769 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p2_low_latency.yaml @@ -0,0 +1,233 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p2` (Low latency) +# Topology: 1 prefill worker(s) x 1 node(s) + 6 decode worker(s) x 1 node(s) +# = 7 nodes / 28 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-low-latency-1p6d-dep4-tp4-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p2 + computeDomain: + name: dsv4-p2-domain + numNodes: 7 + resourceClaimTemplateName: dsv4-p2-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 1 + replicas: 6 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p3_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p3_mid_curve.yaml new file mode 100644 index 00000000..cdd1e4c4 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p3_mid_curve.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p3` (Mid-curve) +# Topology: 1 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 2 node(s) +# = 3 nodes / 12 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-mid-curve-1p1d-dep4-dep8-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p3 + computeDomain: + name: dsv4-p3-domain + numNodes: 3 + resourceClaimTemplateName: dsv4-p3-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 2 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "2048" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p4_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p4_mid_curve.yaml new file mode 100644 index 00000000..dc3ab55b --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p4_mid_curve.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p4` (Mid-curve) +# Topology: 1 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 4 node(s) +# = 5 nodes / 20 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-mid-curve-1p1d-dep4-dep16-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p4 + computeDomain: + name: dsv4-p4-domain + numNodes: 5 + resourceClaimTemplateName: dsv4-p4-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 4 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "2048" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p5_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p5_mid_curve.yaml new file mode 100644 index 00000000..0b10e5d6 --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p5_mid_curve.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p5` (Mid-curve) +# Topology: 2 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 2 node(s) +# = 4 nodes / 16 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-mid-curve-2p1d-dep4-dep8-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p5 + computeDomain: + name: dsv4-p5-domain + numNodes: 4 + resourceClaimTemplateName: dsv4-p5-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 2 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "4096" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 2 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p6_mid_curve.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p6_mid_curve.yaml new file mode 100644 index 00000000..0fa4a9ea --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p6_mid_curve.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p6` (Mid-curve) +# Topology: 4 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 2 node(s) +# = 6 nodes / 24 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-mid-curve-4p1d-dep4-dep8-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p6 + computeDomain: + name: dsv4-p6-domain + numNodes: 6 + resourceClaimTemplateName: dsv4-p6-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 2 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "4096" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 4 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p7_max_throughput.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p7_max_throughput.yaml new file mode 100644 index 00000000..ed220b8c --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p7_max_throughput.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p7` (Max throughput) +# Topology: 6 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 2 node(s) +# = 8 nodes / 32 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-high-conc-6p1d-dep4-dep8-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p7 + computeDomain: + name: dsv4-p7-domain + numNodes: 8 + resourceClaimTemplateName: dsv4-p7-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 2 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "4096" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 6 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p8_max_throughput.yaml b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p8_max_throughput.yaml new file mode 100644 index 00000000..8dd900fb --- /dev/null +++ b/inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep/deepseek-v4-pro/dsv4_p8_max_throughput.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP — operating point `p8` (Max throughput) +# Topology: 8 prefill worker(s) x 1 node(s) + 1 decode worker(s) x 2 node(s) +# = 10 nodes / 40 GB300 GPUs inside one NVL72 ComputeDomain +# Source recipe: NVIDIA srt-slurm `disagg-high-conc-8p1d-dep4-dep8-mtp` +# Pair with: src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml +# src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml + +dynamo: + namespace: dynamo-cloud + releaseVersion: "1.0.1" + deploymentName: dsv4-p8 + computeDomain: + name: dsv4-p8-domain + numNodes: 10 + resourceClaimTemplateName: dsv4-p8-compute-domain-channel + # KV transfer rides the NVL72 fabric (MNNVL); no RDMA NIC resource claim is requested. + serviceAccountName: + frontend: + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.1 + replicas: 9 + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 150 + failureThreshold: 100 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 300 + failureThreshold: 100 + decodeWorker: + nodeCount: 2 + replicas: 1 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "4096" + livenessProbe: + initialDelaySeconds: 600 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + prefillWorker: + nodeCount: 1 + replicas: 8 + envs: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: HF_TOKEN + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "1" + - name: LD_LIBRARY_PATH + value: "/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/plugins:/usr/local/nvidia/lib64" + - name: PYTHONUNBUFFERED + value: "1" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + - name: TP_SOCKET_IFNAME + value: "eth0" + - name: UCX_TLS + value: "cuda_copy,cuda_ipc,tcp" + - name: UCX_CUDA_IPC_ENABLE_MNNVL + value: "y" + - name: UCX_MEMTYPE_CACHE + value: "n" + - name: UCX_MEMTYPE_REG_WHOLE + value: "n" + - name: MC_FORCE_MNNVL + value: "1" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_MNNVL_ENABLE + value: "1" + - name: NCCL_CUMEM_ENABLE + value: "1" + - name: SGLANG_MOONCAKE_CUSTOM_MEM_POOL + value: "True" + - name: SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE + value: "100000" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "100000" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000" + - name: SGLANG_RADIX_DISABLE_REUSE + value: "1" + - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP + value: "1" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV4_REASONING_EFFORT + value: "max" + - name: SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT + value: "1" + - name: SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW + value: "1" + - name: SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN + value: "1" + - name: SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "9216" + livenessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + readinessProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 60 + startupProbe: + initialDelaySeconds: 3000 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 1800 + +secrets: + ngc: + secretName: nvcr-secret + huggingface: + secretName: hf-token-secret + secretData: + token: "hf_api_token" + +volumes: + useGcs: true + gcsfuse: + bucketName: + ssdMountPath: "/ssd" + gcsMounts: + mountPath: "/data/model" + +service: + type: ClusterIP + ports: + frontend: 8000 + worker: 9090 + +workload: + model: deepseek-ai/DeepSeek-V4-Pro + image: + framework: sglang + configFile: serving-args.yaml + configPath: /workload/configs + +network: + subnetworks: [] + gibVersion: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.7 + ncclSettings: + - name: NCCL_DEBUG + value: "VERSION" + +quantizations: + - "fp4" diff --git a/src/docker/dynamo-sglang-deepseek/Dockerfile b/src/docker/dynamo-sglang-deepseek/Dockerfile new file mode 100644 index 00000000..7cf77e2a --- /dev/null +++ b/src/docker/dynamo-sglang-deepseek/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Worker/frontend image for the DeepSeek-R1 Pareto-sweep recipes +# (inference/a4xmax/disaggregated-serving/dynamo/pareto-sweep). +# +# Reproduces the software stack the InferenceX GB300 recipes pin: an SGLang +# release image plus a released ai-dynamo wheel. Build for linux/arm64 (Grace). +# +# DeepSeek-R1 NVFP4: BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130-runtime DYNAMO_VERSION=0.8.1 +# DeepSeek-R1 FP8: BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130 DYNAMO_VERSION=0.8.0 +# +# docker buildx build --platform linux/arm64 \ +# --build-arg BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130-runtime \ +# --build-arg DYNAMO_VERSION=0.8.1 \ +# -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/dynamo-sglang-dsr1:0.8.1 --push . +ARG BASE_IMAGE=lmsysorg/sglang:v0.5.8.post1-cu130-runtime +FROM ${BASE_IMAGE} +ARG BASE_IMAGE + +ARG DYNAMO_VERSION=0.8.1 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +# NIXL ships its shared libraries inside the Python package; expose them at the +# path the Dynamo/UCX plugins expect. Link only directories that exist and log +# the result, so a base image without NIXL is visible in the build output +# instead of failing silently. DeepSeek-R1 transfers KV with NIXL and needs these +# libraries; DeepSeek-V4-Pro uses Mooncake and does not. +RUN mkdir -p /opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu \ + && for dir in /usr/local/lib/python3.*/dist-packages/.nixl_cu13.mesonpy.libs; do \ + if [ -d "$dir" ]; then \ + ln -sf "$dir"/*.so* /opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/; \ + fi; \ + done \ + && if ls /opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/*.so* >/dev/null 2>&1; then \ + echo "NIXL libraries linked:" && ls /opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu/; \ + else \ + echo "WARNING: no NIXL shared libraries found in ${BASE_IMAGE}; NIXL KV transfer will not work"; \ + fi + +RUN pip install --no-cache-dir \ + "ai-dynamo-runtime==${DYNAMO_VERSION}" \ + "ai-dynamo==${DYNAMO_VERSION}" \ + hf_transfer + +# Sanity check: the Dynamo SGLang worker and frontend modules import. +RUN python3 -c "import dynamo.sglang, dynamo.frontend; print('dynamo', '${DYNAMO_VERSION}', 'ok')" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml new file mode 100644 index 00000000..eeb66295 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point ll (Low latency) | srt-slurm recipe `gb300-fp4/8k1k/low_latency` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_trtllm" +fp4-gemm-backend: "flashinfer_trtllm" +tensor-parallel-size: "4" +expert-parallel-size: "1" +prefill-round-robin-balance: true +mem-fraction-static: "0.95" +chunked-prefill-size: "8192" +cuda-graph-max-bs: "128" +scheduler-recv-interval: "10" +enable-symm-mem: true diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml new file mode 100644 index 00000000..4297c1f7 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-low-latency-1p4d-tp4-tp4-prefill.yaml @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point ll (Low latency) | srt-slurm recipe `gb300-fp4/8k1k/low_latency` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_trtllm" +fp4-gemm-backend: "flashinfer_trtllm" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +mem-fraction-static: "0.95" +max-total-tokens: "32768" +chunked-prefill-size: "24576" +cuda-graph-max-bs: "256" +max-running-requests: "512" +scheduler-recv-interval: "10" +enable-symm-mem: true +load-balance-method: "round_robin" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml new file mode 100644 index 00000000..a92e8735 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-decode.yaml @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point max (Max throughput) | srt-slurm recipe `gb300-fp4/8k1k/max_tpt` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_cutedsl" +fp4-gemm-backend: "flashinfer_cutlass" +tensor-parallel-size: "32" +data-parallel-size: "32" +expert-parallel-size: "32" +enable-dp-attention: true +enable-dp-lm-head: true +moe-dense-tp-size: "1" +disable-chunked-prefix-cache: true +decode-log-interval: "1000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +moe-a2a-backend: "deepep" +deepep-mode: "low_latency" +ep-dispatch-algorithm: "static" +ep-num-redundant-experts: "32" +prefill-round-robin-balance: true +mem-fraction-static: "0.83" +max-total-tokens: "524288" +chunked-prefill-size: "24576" +max-running-requests: "16384" +cuda-graph-max-bs: "512" +num-reserved-decode-tokens: "112" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml new file mode 100644 index 00000000..00ecac6f --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-max-throughput-10p1d-tp4-dep32-prefill.yaml @@ -0,0 +1,50 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point max (Max throughput) | srt-slurm recipe `gb300-fp4/8k1k/max_tpt` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_trtllm" +fp4-gemm-backend: "flashinfer_cutlass" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +disable-chunked-prefix-cache: true +decode-log-interval: "1000" +disable-shared-experts-fusion: true +mem-fraction-static: "0.95" +max-total-tokens: "131072" +max-prefill-tokens: "524288" +chunked-prefill-size: "131072" +max-running-requests: "30000" +load-balance-method: "round_robin" +disable-cuda-graph: true diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml new file mode 100644 index 00000000..671f90c3 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-decode.yaml @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point mid (Mid-curve) | srt-slurm recipe `gb300-fp4/8k1k/mid_curve` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_cutedsl" +fp4-gemm-backend: "flashinfer_cutlass" +tensor-parallel-size: "48" +data-parallel-size: "48" +expert-parallel-size: "48" +enable-dp-attention: true +enable-dp-lm-head: true +moe-dense-tp-size: "1" +disable-chunked-prefix-cache: true +decode-log-interval: "1000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +moe-a2a-backend: "deepep" +deepep-mode: "low_latency" +ep-dispatch-algorithm: "static" +ep-num-redundant-experts: "32" +prefill-round-robin-balance: true +mem-fraction-static: "0.83" +max-total-tokens: "524288" +chunked-prefill-size: "24576" +max-running-requests: "16384" +cuda-graph-max-bs: "512" +num-reserved-decode-tokens: "112" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml new file mode 100644 index 00000000..0aa1a8fc --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp4-mid-curve-6p1d-tp4-dep48-prefill.yaml @@ -0,0 +1,50 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 NVFP4 | point mid (Mid-curve) | srt-slurm recipe `gb300-fp4/8k1k/mid_curve` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +quantization: "modelopt_fp4" +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +stream-interval: "50" +watchdog-timeout: "1000000" +context-length: "9600" +moe-runner-backend: "flashinfer_trtllm" +fp4-gemm-backend: "flashinfer_cutlass" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +disable-chunked-prefix-cache: true +decode-log-interval: "1000" +disable-shared-experts-fusion: true +mem-fraction-static: "0.95" +max-total-tokens: "131072" +max-prefill-tokens: "524288" +chunked-prefill-size: "131072" +max-running-requests: "30000" +load-balance-method: "round_robin" +disable-cuda-graph: true diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml new file mode 100644 index 00000000..64a7ab92 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-decode.yaml @@ -0,0 +1,46 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point ll (Low latency) | srt-slurm recipe `gb300-fp8/8k1k/stp/low-latency` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +quantization: "fp8" +moe-runner-backend: "flashinfer_trtllm" +fp8-gemm-backend: "flashinfer_trtllm" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +stream-interval: "10" +mem-fraction-static: "0.85" +chunked-prefill-size: "-1" +cuda-graph-max-bs: "128" +max-running-requests: "128" +scheduler-recv-interval: "1" +prefill-round-robin-balance: true diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml new file mode 100644 index 00000000..a8e61667 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-low-latency-1p1d-tp4-tp4-prefill.yaml @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point ll (Low latency) | srt-slurm recipe `gb300-fp8/8k1k/stp/low-latency` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +quantization: "fp8" +moe-runner-backend: "flashinfer_trtllm" +fp8-gemm-backend: "flashinfer_trtllm" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +stream-interval: "10" +mem-fraction-static: "0.95" +max-total-tokens: "32768" +chunked-prefill-size: "32768" +max-prefill-tokens: "32768" +cuda-graph-max-bs: "128" +max-running-requests: "128" +load-balance-method: "round_robin" +scheduler-recv-interval: "10" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml new file mode 100644 index 00000000..95610ba9 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-decode.yaml @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point max (Max throughput) | srt-slurm recipe `gb300-fp8/8k1k/stp/max` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +skip-tokenizer-init: true +tensor-parallel-size: "24" +data-parallel-size: "24" +expert-parallel-size: "24" +enable-dp-attention: true +enable-dp-lm-head: true +stream-interval: "50" +decode-log-interval: "1000" +max-running-requests: "45000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +mem-fraction-static: "0.82" +chunked-prefill-size: "36864" +moe-a2a-backend: "deepep" +deepep-mode: "low_latency" +ep-dispatch-algorithm: "static" +moe-dense-tp-size: "1" +prefill-round-robin-balance: true +ep-num-redundant-experts: "32" +deepep-config: '{"normal_dispatch": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 28, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 256}, "normal_combine": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 15, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 128}}' +cuda-graph-bs: "[1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328, 336, 344, 352, 360, 368, 376, 384, 416, 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768]" +cuda-graph-max-bs: "768" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml new file mode 100644 index 00000000..09bb3f22 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-max-throughput-6p1d-dep8-dep24-prefill.yaml @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point max (Max throughput) | srt-slurm recipe `gb300-fp8/8k1k/stp/max` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +skip-tokenizer-init: true +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +stream-interval: "50" +max-running-requests: "30000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +mem-fraction-static: "0.75" +max-total-tokens: "524288" +chunked-prefill-size: "131072" +load-balance-method: "round_robin" +disable-cuda-graph: true +moe-a2a-backend: "deepep" +deepep-mode: "normal" +ep-dispatch-algorithm: "dynamic" +moe-dense-tp-size: "1" +ep-num-redundant-experts: "32" +deepep-config: '{"normal_dispatch": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 28, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 256}, "normal_combine": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 15, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 128}}' diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml new file mode 100644 index 00000000..5b4bb523 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-decode.yaml @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point mid (Mid-curve) | srt-slurm recipe `gb300-fp8/8k1k/stp/mid` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +skip-tokenizer-init: true +tensor-parallel-size: "32" +data-parallel-size: "32" +expert-parallel-size: "32" +enable-dp-attention: true +enable-dp-lm-head: true +stream-interval: "50" +decode-log-interval: "1000" +max-running-requests: "45000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +mem-fraction-static: "0.82" +chunked-prefill-size: "36864" +moe-a2a-backend: "deepep" +deepep-mode: "low_latency" +ep-dispatch-algorithm: "static" +moe-dense-tp-size: "1" +prefill-round-robin-balance: true +ep-num-redundant-experts: "32" +deepep-config: '{"normal_dispatch": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 28, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 256}, "normal_combine": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 15, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 128}}' +cuda-graph-bs: "[1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328, 336, 344, 352, 360, 368, 376, 384, 416, 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768]" +cuda-graph-max-bs: "768" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml new file mode 100644 index 00000000..27d485fb --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsr1-fp8-mid-curve-5p1d-dep8-dep32-prefill.yaml @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-R1-0528 FP8 | point mid (Mid-curve) | srt-slurm recipe `gb300-fp8/8k1k/stp/mid` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-R1" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "nixl" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +kv-cache-dtype: "fp8_e4m3" +attention-backend: "trtllm_mla" +disable-radix-cache: true +watchdog-timeout: "1000000" +context-length: "9300" +skip-tokenizer-init: true +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +stream-interval: "50" +max-running-requests: "30000" +disable-shared-experts-fusion: true +eplb-algorithm: "deepseek" +mem-fraction-static: "0.75" +max-total-tokens: "524288" +chunked-prefill-size: "131072" +load-balance-method: "round_robin" +disable-cuda-graph: true +moe-a2a-backend: "deepep" +deepep-mode: "normal" +ep-dispatch-algorithm: "dynamic" +moe-dense-tp-size: "1" +ep-num-redundant-experts: "32" +deepep-config: '{"normal_dispatch": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 28, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 256}, "normal_combine": {"num_sms": 128, "num_max_nvl_chunked_send_tokens": 15, "num_max_nvl_chunked_recv_tokens": 256, "num_max_rdma_chunked_send_tokens": 6, "num_max_rdma_chunked_recv_tokens": 128}}' diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml new file mode 100644 index 00000000..4bf16ed3 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-decode.yaml @@ -0,0 +1,43 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p1 (Low latency) | srt-slurm recipe `disagg-low-latency-1p1d-tp4-tp4-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +moe-runner-backend: "flashinfer_mxfp4" +disable-flashinfer-autotune: true +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.9" +max-running-requests: "8" +cuda-graph-max-bs: "8" +swa-full-tokens-ratio: "0.1" +context-length: "16384" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml new file mode 100644 index 00000000..dd804f57 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p1-low-latency-1p1d-tp4-tp4-prefill.yaml @@ -0,0 +1,38 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p1 (Low latency) | srt-slurm recipe `disagg-low-latency-1p1d-tp4-tp4-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +moe-runner-backend: "flashinfer_mxfp4" +disable-flashinfer-autotune: true +mem-fraction-static: "0.9" +max-running-requests: "8" +cuda-graph-max-bs: "8" +chunked-prefill-size: "32768" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml new file mode 100644 index 00000000..da2c1f96 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-decode.yaml @@ -0,0 +1,43 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p2 (Low latency) | srt-slurm recipe `disagg-low-latency-1p6d-dep4-tp4-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "1" +expert-parallel-size: "1" +moe-runner-backend: "flashinfer_mxfp4" +disable-flashinfer-autotune: true +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.9" +max-running-requests: "128" +cuda-graph-max-bs: "128" +swa-full-tokens-ratio: "0.1" +context-length: "16384" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml new file mode 100644 index 00000000..6bf2a997 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p2-low-latency-1p6d-dep4-tp4-prefill.yaml @@ -0,0 +1,39 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p2 (Low latency) | srt-slurm recipe `disagg-low-latency-1p6d-dep4-tp4-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "128" +cuda-graph-max-bs: "128" +chunked-prefill-size: "32768" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml new file mode 100644 index 00000000..239a5715 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p3 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-1p1d-dep4-dep8-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.94" +max-running-requests: "3072" +cuda-graph-max-bs: "256" +swa-full-tokens-ratio: "0.15" +context-length: "16384" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml new file mode 100644 index 00000000..548bbb77 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p3-mid-curve-1p1d-dep4-dep8-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p3 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-1p1d-dep4-dep8-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "256" +cuda-graph-max-bs: "256" +chunked-prefill-size: "32768" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml new file mode 100644 index 00000000..6d6d1d60 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p4 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-1p1d-dep4-dep16-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "16" +data-parallel-size: "16" +expert-parallel-size: "16" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.94" +max-running-requests: "3072" +cuda-graph-max-bs: "256" +swa-full-tokens-ratio: "0.15" +context-length: "16384" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml new file mode 100644 index 00000000..ba49a1de --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p4-mid-curve-1p1d-dep4-dep16-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p4 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-1p1d-dep4-dep16-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "256" +cuda-graph-max-bs: "256" +chunked-prefill-size: "32768" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml new file mode 100644 index 00000000..64e77e60 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p5 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-2p1d-dep4-dep8-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.9" +max-running-requests: "3072" +cuda-graph-max-bs: "512" +swa-full-tokens-ratio: "0.15" +context-length: "16384" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml new file mode 100644 index 00000000..a6f3d270 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p5-mid-curve-2p1d-dep4-dep8-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p5 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-2p1d-dep4-dep8-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "512" +cuda-graph-max-bs: "512" +chunked-prefill-size: "32768" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml new file mode 100644 index 00000000..31cfb546 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p6 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-4p1d-dep4-dep8-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "3" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "4" +mem-fraction-static: "0.9" +max-running-requests: "3072" +cuda-graph-max-bs: "1024" +swa-full-tokens-ratio: "0.15" +context-length: "16384" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml new file mode 100644 index 00000000..60dba64f --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p6-mid-curve-4p1d-dep4-dep8-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p6 (Mid-curve) | srt-slurm recipe `disagg-mid-curve-4p1d-dep4-dep8-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "1024" +cuda-graph-max-bs: "1024" +chunked-prefill-size: "32768" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml new file mode 100644 index 00000000..842d84c1 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p7 (Max throughput) | srt-slurm recipe `disagg-high-conc-6p1d-dep4-dep8-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "2" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "3" +mem-fraction-static: "0.9" +max-running-requests: "4096" +cuda-graph-max-bs: "1024" +swa-full-tokens-ratio: "0.15" +context-length: "9216" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml new file mode 100644 index 00000000..d6dc7fe9 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p7-max-throughput-6p1d-dep4-dep8-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p7 (Max throughput) | srt-slurm recipe `disagg-high-conc-6p1d-dep4-dep8-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "1024" +cuda-graph-max-bs: "1024" +chunked-prefill-size: "32768" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml new file mode 100644 index 00000000..c8d90432 --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-decode.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p8 (Max throughput) | srt-slurm recipe `disagg-high-conc-8p1d-dep4-dep8-mtp` | DECODE worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "decode" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "8" +data-parallel-size: "8" +expert-parallel-size: "8" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +speculative-algo: "EAGLE" +speculative-num-steps: "1" +speculative-eagle-topk: "1" +speculative-num-draft-tokens: "2" +mem-fraction-static: "0.85" +max-running-requests: "8192" +cuda-graph-max-bs: "1280" +swa-full-tokens-ratio: "0.1" +context-length: "9216" +stream-interval: "60" diff --git a/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml b/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml new file mode 100644 index 00000000..1fa62a0e --- /dev/null +++ b/src/frameworks/a4xmax/dynamo-configs/dsv4-p8-max-throughput-8p1d-dep4-dep8-prefill.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DeepSeek-V4-Pro (MXFP4) + EAGLE MTP | point p8 (Max throughput) | srt-slurm recipe `disagg-high-conc-8p1d-dep4-dep8-mtp` | PREFILL worker +# +# Each `key: value` line becomes `--key value` on the dynamo.sglang command line; +# `key: true` becomes a bare `--key` flag. Multi-node rendezvous flags +# (--dist-init-addr/--nnodes/--node-rank) and --model-path are injected by the +# Dynamo operator and the Helm chart launcher, so they are not listed here. + +served-model-name: "deepseek-ai/DeepSeek-V4-Pro" +disaggregation-mode: "prefill" +disaggregation-transfer-backend: "mooncake" +disaggregation-bootstrap-port: "30001" +host: "0.0.0.0" +port: "9090" +trust-remote-code: true +tool-call-parser: "deepseekv4" +tensor-parallel-size: "4" +data-parallel-size: "4" +expert-parallel-size: "4" +enable-dp-attention: true +enable-dp-lm-head: true +moe-a2a-backend: "megamoe" +mem-fraction-static: "0.9" +max-running-requests: "1024" +cuda-graph-max-bs: "1024" +chunked-prefill-size: "32768" +stream-interval: "60"