From 66e05fae097c655883829d28ef04fa1b90d2ace9 Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Mon, 26 Oct 2026 16:51:22 +0000 Subject: [PATCH] Add SGLang-TorchTPU Qwen3-Coder-480B-A35B-Instruct-FP8 single-host recipe for Ironwood (tpu7x 2x2x1) --- .../Qwen3-Coder-480B/single-host/README.md | 321 ++++++++++++++++++ .../single-host/qwen3-coder-480b-server.yaml | 215 ++++++++++++ 2 files changed, 536 insertions(+) create mode 100644 inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/README.md create mode 100644 inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/qwen3-coder-480b-server.yaml diff --git a/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/README.md b/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/README.md new file mode 100644 index 00000000..f4e4f840 --- /dev/null +++ b/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/README.md @@ -0,0 +1,321 @@ +# Serve Qwen3-Coder-480B-A35B-Instruct-FP8 with SGLang (TorchTPU) on Ironwood TPU (Single-Host) + +In this guide, we show how to serve Qwen3-Coder-480B-A35B-Instruct-FP8 (`Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8`, FP8 weights + FP8 KV cache) with [SGLang](https://github.com/sgl-project/sglang) using the **[SGLang-TorchTPU](https://github.com/sgl-project/sglang-torchtpu)** backend on Ironwood (TPU7x) using Google Kubernetes Engine (GKE). + +We deploy it on a single **tpu7x-standard-4t** node pool (`2x2x1` topology, 4 TPU7x chips / 8 TensorCores) using **Tensor Parallelism (TP) size of 8** and **Expert Parallelism (EP) size of 8**. + +## Verified Models + +The following model is verified for single-host deployment on TPU7x with the SGLang-TorchTPU backend. + +| Model | Parameters | Min TPUs (Chips) | Topology | HuggingFace | +| :---- | :---- | :---- | :---- | :---- | +| Qwen3-Coder-480B-A35B-Instruct-FP8 | 480B MoE (35B active) | 4× (`tpu7x-standard-4t`) | `2x2x1` (1 host) | [Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8) | + +> [!NOTE] +> The server flags follow the upstream SGLang-TorchTPU recipe for this model, [`benchmark/tpu/recipe/qwen3_coder_480b/server.sh`](https://github.com/sgl-project/sglang-torchtpu/blob/tpu-v0.1.0/benchmark/tpu/recipe/qwen3_coder_480b/server.sh), sized for 8K-input / 1K-output requests at 64 concurrent sequences. For longer prompts raise `--context-length` and `--max-total-tokens` (64 × context length) together. + +## Prerequisites + +1. A GKE cluster with TPU7x (Ironwood) support. +2. `gcloud` CLI and `kubectl` installed and configured. +3. A Hugging Face account and access token with read access to `Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8`. + +--- + +## Step 1: Define Parameters + +Export the necessary environment variables for your Google Cloud project and GKE cluster: + +```shell +export PROJECT_ID="" +export CLUSTER_NAME="" +export REGION="" +export ZONE="" +export NODEPOOL_NAME="qwen3-coder-480b-tpu7x-pool" +``` + +--- + +## Step 2: Create the Single-Host TPU Node Pool + +Create a TPU7x (Ironwood) node pool with the `tpu7x-standard-4t` machine type and `2x2x1` topology (1 host, 4 chips): + +```shell +gcloud container node-pools create ${NODEPOOL_NAME} \ + --project=${PROJECT_ID} \ + --location=${REGION} \ + --node-locations=${ZONE} \ + --num-nodes=1 \ + --machine-type=tpu7x-standard-4t \ + --tpu-topology=2x2x1 \ + --cluster=${CLUSTER_NAME} +``` + +--- + +## Step 3: Setup Kubernetes Configurations + +1. Configure `kubectl` to communicate with your cluster: + +```shell +gcloud container clusters get-credentials ${CLUSTER_NAME} --location=${REGION} --project=${PROJECT_ID} +``` + +2. Create the isolated namespace `sglang-qwen3-coder-480b`: + +```shell +kubectl create namespace sglang-qwen3-coder-480b +``` + +3. Create a Kubernetes secret for your Hugging Face token in the namespace: + +```shell +export HF_TOKEN="YOUR_HUGGING_FACE_API_TOKEN" +kubectl create secret generic hf-secret \ + --namespace=sglang-qwen3-coder-480b \ + --from-literal=hf_api_token=${HF_TOKEN} +``` + +--- + +## Step 4: Apply the SGLang Server Manifest + +Apply the SGLang server manifest using [qwen3-coder-480b-server.yaml](./qwen3-coder-480b-server.yaml): + +```shell +cd tpu-recipes/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host +kubectl apply -f qwen3-coder-480b-server.yaml +``` + +Monitor the progress of the server deployment: + +```shell +kubectl get pods -n sglang-qwen3-coder-480b -w +``` + +Follow the server logs during model download and TPU compilation (the first start downloads ~480 GB of weights to the attached Hyperdisk, then compiles the decode and prefill programs): + +```shell +kubectl logs -n sglang-qwen3-coder-480b deployment/sglang-qwen3-coder-480b -c sglang-torchtpu -f +``` + +Once the server has loaded the model and completed compilation, you will see: + +```text +INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +The server is fired up and ready to roll! +``` + +--- + +## Step 5: Interact with the Model (Optional) + +1. Port-forward the SGLang service to port `8000` on your local machine: + +```shell +kubectl port-forward -n sglang-qwen3-coder-480b service/sglang-qwen3-coder-480b-service 8000:8000 +``` + +2. Send a test chat completion request: + +```shell +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", + "messages": [ + {"role": "user", "content": "Write a Python function that parses an ISO-8601 timestamp."} + ], + "max_tokens": 256, + "temperature": 0.7 + }' +``` + +--- + +## Step 6: Run Serving Benchmarks + +We use SGLang's built-in serving benchmark (`python3 -m sglang.bench_serving`, shipped in the same image) against the running server across two workloads: **1K Input / 1K Output (`1k/1k`)** and **8K Input / 1K Output (`8k/1k`)**, both at 64 concurrent requests. + +### Workload A: 1K Input / 1K Output (`1k/1k`) + +Save the following manifest as `qwen3-coder-480b-benchmark-1k1k.yaml`: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: qwen3-coder-480b-bench-1k1k + namespace: sglang-qwen3-coder-480b +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 60 + containers: + - name: sglang-bench + image: lmsysorg/sglang:v0.1.0-tpu + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + while ! curl -sf http://sglang-qwen3-coder-480b-service:8000/health; do + sleep 30 + echo "Waiting for sglang-qwen3-coder-480b-service:8000/health..." + done + python3 -m sglang.bench_serving \ + --backend sglang \ + --host sglang-qwen3-coder-480b-service \ + --port 8000 \ + --model Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --random-range-ratio 0.8 \ + --num-prompts 320 \ + --max-concurrency 64 \ + --disable-tqdm + env: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + key: hf_api_token + name: hf-secret + optional: true +``` + +Apply it using `kubectl`: + +```shell +kubectl apply -f qwen3-coder-480b-benchmark-1k1k.yaml +``` + +Check the logs of the benchmark run: + +```shell +kubectl logs -n sglang-qwen3-coder-480b qwen3-coder-480b-bench-1k1k -f +``` + +When complete, clean up the benchmark pod: + +```shell +kubectl delete -f qwen3-coder-480b-benchmark-1k1k.yaml +``` + +--- + +### Workload B: 8K Input / 1K Output (`8k/1k`) + +Save the following manifest as `qwen3-coder-480b-benchmark-8k1k.yaml`: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: qwen3-coder-480b-bench-8k1k + namespace: sglang-qwen3-coder-480b +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 60 + containers: + - name: sglang-bench + image: lmsysorg/sglang:v0.1.0-tpu + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + while ! curl -sf http://sglang-qwen3-coder-480b-service:8000/health; do + sleep 30 + echo "Waiting for sglang-qwen3-coder-480b-service:8000/health..." + done + python3 -m sglang.bench_serving \ + --backend sglang \ + --host sglang-qwen3-coder-480b-service \ + --port 8000 \ + --model Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 0.8 \ + --num-prompts 320 \ + --max-concurrency 64 \ + --disable-tqdm + env: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + key: hf_api_token + name: hf-secret + optional: true +``` + +Apply it using `kubectl`: + +```shell +kubectl apply -f qwen3-coder-480b-benchmark-8k1k.yaml +``` + +Check the logs of the benchmark run: + +```shell +kubectl logs -n sglang-qwen3-coder-480b qwen3-coder-480b-bench-8k1k -f +``` + +When complete, clean up the benchmark pod: + +```shell +kubectl delete -f qwen3-coder-480b-benchmark-8k1k.yaml +``` + +--- + +### Example Benchmark Output + +The benchmark client will print a table resembling the following: + +```text +============ Serving Benchmark Result ============ +Backend: sglang +Successful requests: xxx +Benchmark duration (s): xxx +Total input tokens: xxx +Total generated tokens: xxx +Request throughput (req/s): xxx +Input token throughput (tok/s): xxx +Output token throughput (tok/s): xxx +Total token throughput (tok/s): xxx +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): xxx +Median E2E Latency (ms): xxx +---------------Time to First Token---------------- +Mean TTFT (ms): xxx +Median TTFT (ms): xxx +P99 TTFT (ms): xxx +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): xxx +Median TPOT (ms): xxx +P99 TPOT (ms): xxx +---------------Inter-Token Latency---------------- +Mean ITL (ms): xxx +Median ITL (ms): xxx +P99 ITL (ms): xxx +================================================== +``` + +--- + +## Step 7: Cleanup + +1. Delete the SGLang server resources: + +```shell +kubectl delete -f qwen3-coder-480b-server.yaml +``` + +2. (Optional) Delete the GKE TPU node pool: + +```shell +gcloud container node-pools delete ${NODEPOOL_NAME} \ + --cluster=${CLUSTER_NAME} \ + --location=${REGION} \ + --project=${PROJECT_ID} +``` diff --git a/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/qwen3-coder-480b-server.yaml b/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/qwen3-coder-480b-server.yaml new file mode 100644 index 00000000..72566aaa --- /dev/null +++ b/inference/ironwood/SGLang/SGLang-TorchTPU/Qwen3-Coder-480B/single-host/qwen3-coder-480b-server.yaml @@ -0,0 +1,215 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hyperdisk-balanced-tpu +provisioner: pd.csi.storage.gke.io +parameters: + type: hyperdisk-balanced +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hf-cache-pvc + namespace: sglang-qwen3-coder-480b +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1000Gi + storageClassName: hyperdisk-balanced-tpu +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sglang-qwen3-coder-480b + namespace: sglang-qwen3-coder-480b +spec: + replicas: 1 + # TPU node pools are fixed-size: a RollingUpdate needs a spare node to surge + # onto, so use Recreate to avoid rollout deadlocks. + strategy: + type: Recreate + selector: + matchLabels: + app: sglang-qwen3-coder-480b + template: + metadata: + labels: + app: sglang-qwen3-coder-480b + spec: + terminationGracePeriodSeconds: 75 + nodeSelector: + cloud.google.com/gke-tpu-accelerator: tpu7x + cloud.google.com/gke-tpu-topology: 2x2x1 + tolerations: + - key: "google.com/tpu" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: sglang-torchtpu + image: lmsysorg/sglang:v0.1.0-tpu + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + mkdir -p "${HF_HOME}" "${TORCH_TPU_INTERNAL_TIER3_COMPILATION_CACHE_ROOT}" + # Flags follow the upstream sglang-torchtpu recipe for this model on one + # tpu7x host (benchmark/tpu/recipe/qwen3_coder_480b/server.sh): 4 chips x + # 2 TensorCores = TP 8, the 160 experts split EP 8, FP8 weights + fp8_e4m3 KV + # cache. Sized for 8k-input / 1k-output requests at 64 concurrent sequences + # (context 9216, KV pool 64 x 9216 tokens); raise --context-length and + # --max-total-tokens together for longer prompts. + exec python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 \ + --host 0.0.0.0 \ + --port 8000 \ + --device tpu \ + --dtype bfloat16 \ + --tp-size 8 \ + --ep-size 8 \ + --attention-backend pallas_tpu_rpa_batched \ + --grammar-backend none \ + --quantization fp8 \ + --kv-cache-dtype fp8_e4m3 \ + --tpu-fp8-runtime channelwise \ + --page-size 256 \ + --mem-fraction-static 0.8 \ + --max-running-requests 64 \ + --torch-compile-max-bs 64 \ + --context-length 9216 \ + --chunked-prefill-size 8192 \ + --max-prefill-tokens 8192 \ + --max-total-tokens 589824 \ + --enable-torch-compile \ + --tpu-compile-tracer direct \ + --sampling-defaults openai \ + --enable-fused-qk-norm-rope \ + --disable-radix-cache \ + --watchdog-timeout 3600 + env: + - name: HF_HOME + value: /data/huggingface + - name: HUGGING_FACE_HUB_TOKEN + valueFrom: + secretKeyRef: + name: hf-secret + key: hf_api_token + optional: true + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-secret + key: hf_api_token + optional: true + # Static TPU serving environment from sglang-torchtpu benchmark/tpu/common_env.sh. + - name: SGLANG_PLATFORM + value: "tpu" + - name: SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION + value: "0" + - name: SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN + value: "1" + - name: SGLANG_TPU_OVERLAP_PROFILE + value: "1" + # MoE: dense matmul for small permute / combine (upstream recipe value). + - name: SGLANG_TPU_FUSED_MOE_PERMUTE_DENSE_THRESHOLD + value: "2048" + - name: TORCH_TPU_INTERNAL_MATERIALIZE_COLLECTIVE_TENSORS + value: "false" + # The batched RPA kernel needs dynamic SMEM negotiation. + - name: TORCH_TPU_INTERNAL_XLA_OPTIONS + value: "xla_tpu_use_dynamic_smem_negotiation=true" + # Persist compiled TPU programs on the data disk across restarts. + - name: TORCH_TPU_INTERNAL_TIER2_COMPILATION_CACHE + value: "torchtpu_qwen3_coder_480b" + - name: TORCH_TPU_INTERNAL_TIER3_COMPILATION_CACHE_ROOT + value: /data/torch_tpu_compile_cache + - name: TORCH_TPU_INTERNAL_TIER3_COMPILATION_CACHE_LOCAL_BACKUP_TASK + value: "1" + - name: TORCH_LOGS + value: "+recompiles" + - name: TPU_ENABLE_DIAGNOSTIC_COORDINATOR + value: "0" + - name: TPU_DISABLE_DIAGNOSTIC_COORDINATOR + value: "1" + - name: TPU_ENABLE_TELEMETRY_HARVESTING + value: "0" + - name: TPU_ENABLE_TELEMETRY + value: "0" + - name: ENABLE_TPU_TELEMETRY + value: "0" + - name: TPU_TELEMETRY_INTERVAL_SECONDS + value: "86400" + - name: TPU_TELEMETRY_INTERVAL_SEC + value: "86400" + - name: TPU_DIAGNOSTIC_HARVESTER_INTERVAL_SEC + value: "86400" + - name: LIBTPU_INIT_ARGS + value: "--xla_tpu_enable_async_collective_fusion=true --xla_tpu_overlap_compute_collective_tc=true --xla_tpu_enable_async_collective_fusion_multiple_steps=true --xla_tpu_enable_latency_hiding_scheduler=true --xla_latency_hiding_scheduler_rerun=10 --xla_tpu_all_gather_collective_matmul_mode=post_spmd_conservative --xla_tpu_reduce_scatter_collective_matmul_mode=post_spmd_conservative --xla_tpu_enable_sparse_core_collective_offload_all_reduce=false --xla_tpu_offload_gather_to_sparsecore=false --xla_tpu_enable_sparse_core_collective_offload_reduce_scatter=false" + ports: + - name: http + containerPort: 8000 + protocol: TCP + resources: + limits: + memory: 800Gi + google.com/tpu: "4" + requests: + cpu: "64" + memory: 600Gi + google.com/tpu: "4" + # Downloading ~480 GB of weights and compiling the TPU programs takes a while + # on first start; allow up to 4 hours before the pod is considered failed. + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 480 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + volumeMounts: + - name: hf-cache + mountPath: /data + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 100Gi + - name: hf-cache + persistentVolumeClaim: + claimName: hf-cache-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-qwen3-coder-480b-service + namespace: sglang-qwen3-coder-480b +spec: + selector: + app: sglang-qwen3-coder-480b + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 8000 + targetPort: 8000