From e7182451a4a355f4475ef38776daacdf9eabbbe1 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 13:51:51 -0700
Subject: [PATCH 01/10] docs(verification): add Qwen 3.8 27B GB200 card

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../qwen3.8-27b/card.yaml                     | 430 ++++++++++++++++++
 1 file changed, 430 insertions(+)
 create mode 100644 examples/model_verification_cards/qwen3.8-27b/card.yaml

diff --git a/examples/model_verification_cards/qwen3.8-27b/card.yaml b/examples/model_verification_cards/qwen3.8-27b/card.yaml
new file mode 100644
index 0000000000..69109becb1
--- /dev/null
+++ b/examples/model_verification_cards/qwen3.8-27b/card.yaml
@@ -0,0 +1,430 @@
+# Agent-readable model verification card.
+# status: unverified | verified | unsupported | not_applicable
+
+title: qwen3_8_27b
+summary: >
+  Performance disclaimer: this model has not been performance-tuned; reported
+  timing and throughput metrics are sanity checks, not optimized performance
+  results. All workloads in this card target GB200 hardware. Conversion and
+  deterministic inference and all required functional training gates are
+  verified. The unchanged model-comparison script is currently incompatible
+  with the Qwen 3.8 processor because it does not forward mm_token_type_ids.
+  No dense-27B GB200 performance or Megatron FSDP recipe is available.
+verification_index:
+  model_level:
+    verified:
+      - hf_to_megatron_cpu
+      - hf_to_megatron_gpu
+      - megatron_to_hf_cpu
+      - megatron_to_hf_gpu
+      - inference
+    unverified: [manual_forward_pass]
+  training:
+    GB200:
+      verified: [pretrain, sft, sft_export_inference, sft_long_context, peft, checkpoint_resume]
+model:
+  hf_id: Qwen/Qwen3.8-27B
+  hf_revision: 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0  # pragma: allowlist secret
+  architecture: Qwen3_5ForConditionalGeneration
+  min_transformers_version: "5.8.0"
+verification_environment:
+  base_container: nvcr.io/nvidia/nemo:26.08.rc5
+  bridge_commit: d790e32a88654968021291d68c510f41d64df461  # pragma: allowlist secret
+
+items:
+  hf_to_megatron_cpu:
+    status: verified
+    precision: bf16
+    command: >
+      ./scripts/conversion/convert.sh import --executor slurm --device cpu
+      --nodes 1 --gpus-per-node 1 --mem 512G
+      --hf-model Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --megatron-path work/model-verification/qwen3.8-27b/cpu-megatron
+    last_verified: 2026-08-14
+    expected_result: >
+      The pinned-revision CPU import completed in 3m11s and created a reloadable
+      iter_0000000 containing 8 nonempty files totaling 55,586,366,001 bytes.
+      The converter selected Qwen35VLBridge, completed all 956 mappings, kept
+      model weights on CPU, and reported 27,781,427,952 parameters. One shared
+      runtime GB200 was used for Transformer Engine vision-module construction.
+
+  hf_to_megatron_gpu:
+    status: verified
+    precision: bf16
+    command: >
+      ./scripts/conversion/convert.sh import --executor slurm --device gpu
+      --nodes 1 --gpus-per-node 4
+      --hf-model Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --megatron-path work/model-verification/qwen3.8-27b/gpu-megatron
+      --tp 4
+    last_verified: 2026-08-14
+    expected_result: >
+      The pinned-revision four-GB200 import completed in 2m01s at TP4/PP1 and
+      created an 11-file iter_0000000 totaling 55,592,033,255 bytes, including
+      four distributed checkpoint shards. All 956 conversion mappings completed,
+      and subsequent export and two independent inference runs reloaded it.
+
+  megatron_to_hf_cpu:
+    status: verified
+    precision: bf16
+    command: >
+      ./scripts/conversion/convert.sh export --executor slurm --device cpu
+      --nodes 1 --gpus-per-node 1 --mem 512G
+      --hf-model Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --megatron-path
+      work/model-verification/qwen3.8-27b/cpu-megatron/iter_0000000
+      --hf-path work/model-verification/qwen3.8-27b/cpu-hf-export
+    last_verified: 2026-08-14
+    expected_result: >
+      CPU export completed in 2m16s, wrote every source tensor, and produced 26
+      files totaling 55,583,128,197 bytes. The index preserves the original
+      1,199-key set and exact 18-shard placement with 55,562,855,904 BF16 tensor
+      bytes. Transformers strictly reloaded the export as
+      Qwen3_5ForConditionalGeneration with empty missing, unexpected,
+      mismatched, and error sets, then completed processor-native GB200
+      inference. Export normalizes vision_config.model_type from qwen3_5 to
+      qwen3_5_vision and updates producer-version metadata without changing the
+      architecture.
+
+  megatron_to_hf_gpu:
+    status: verified
+    precision: bf16
+    command: >
+      ./scripts/conversion/convert.sh export --executor slurm --device gpu
+      --nodes 1 --gpus-per-node 4
+      --hf-model Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --megatron-path
+      work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+      --hf-path work/model-verification/qwen3.8-27b/gpu-hf-export
+      --tp 4
+    last_verified: 2026-08-14
+    expected_result: >
+      The four-GB200 distributed export completed in 1m55s, wrote every source
+      tensor, and produced 26 files totaling 55,583,128,222 bytes. Its index has
+      the same 1,199 keys, 18-shard placement, and 55,562,855,904 BF16 tensor
+      bytes as the pinned source and CPU export. Transformers strictly reloaded
+      it with empty missing, unexpected, mismatched, and error sets and completed
+      processor-native inference. Relative to CPU export, config.json only adds
+      the metadata field vision_config.dtype=bfloat16.
+
+  manual_forward_pass:
+    status: unverified
+    precision: bf16
+    command: >
+      ./scripts/inference/infer.sh --nodes 1 --gpus-per-node 4
+      --task model-comparison
+      --hf_model_path Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --model_class Qwen3_5ForConditionalGeneration
+      --megatron_model_path
+      work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+      --tp 4 --image_path work/data/medpix/verification.png
+      --prompt "What was observed in the imaging?"
+    last_verified: null
+    expected_result: >
+      The unchanged command loaded the pinned HF weights and prepared a
+      128-token input plus a 256-by-1536 pixel tensor, but Transformers rejected
+      the forward before logits were produced: image_grid_thw was present while
+      the newly required mm_token_type_ids input was absent. The processor emits
+      that field, but the unchanged comparison script does not forward it.
+      Therefore no next-token, cosine-similarity, or absolute-logit-difference
+      claim is made. The input was MedPix-VQA revision
+      69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0, case MPX1095;
+      its 256-by-256 image has SHA-256
+      f5210fcff6dad525c80fcd0f462b274c9da4d634ccd4f1cc76578f1bfe13646d.
+
+  inference:
+    status: verified
+    precision: bf16
+    command: >
+      ./scripts/inference/infer.sh --nodes 1 --gpus-per-node 4
+      --task vlm-generation
+      --hf_model_path Qwen/Qwen3.8-27B
+      --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+      --megatron_model_path
+      work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+      --tp 4 --image_path work/data/medpix/verification.png
+      --prompt "What was observed in the imaging?" --max_new_tokens 32
+    last_verified: 2026-08-14
+    expected_result: >
+      Two independent four-GB200 greedy-decoding runs loaded the imported TP4
+      checkpoint and consumed MedPix-VQA revision
+      69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0, case MPX1095,
+      using source-image SHA-256
+      f5210fcff6dad525c80fcd0f462b274c9da4d634ccd4f1cc76578f1bfe13646d.
+      Both exited successfully with byte-identical 32-token completions and
+      reached the 32-token maximum without EOS. The exact completion was: "The
+      user has provided an axial brain MRI image, likely T2-weighted given the
+      bright CSF and the appearance of the structures. The image is from Med".
+
+  pretrain:
+    GB200:
+      status: verified
+      precision: bf16
+      enabled_features: {}
+      command: >
+        ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
+        --recipe qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config
+        --mode pretrain --deterministic
+        --pretrained_checkpoint
+        work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+        --max_steps 20 --warmup_iters 2
+        --save_dir work/model-verification/qwen3.8-27b/pretrain/checkpoints
+        --save_interval 10
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        checkpoint.load=null validation.eval_iters=0 validation.eval_interval=0
+        ddp.check_for_large_grads=true logger.log_interval=1
+        logger.log_throughput=true
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain/resolved-config.yaml
+        scheduler.lr_decay_iters=20
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 6.858752
+        final_loss: 5.000320
+        last_10_steps_step_time_ms_avg: 39373.700
+        last_10_steps_model_tflops_per_gpu_avg: 35.640
+      expected_result: >
+        The 16-GB200 projection-only MockVLM pretrain completed exactly 20
+        deterministic steps at TP4/PP4, sequence length 4096, and MBS2/GBS32.
+        Language and vision weights were frozen while the multimodal projector
+        trained. Loss remained finite from 6.858752 to 5.000320 with zero
+        skipped or NaN iterations. Steps 11-20 averaged 39,373.700 ms and
+        35.640 model TFLOP/s/GPU. Checkpoints at iterations 10 and 20 each
+        contain 20 files totaling 56,110,835,009 bytes, the tracker records
+        iteration 20, and both built-in 32-iteration validation and test passes
+        completed at loss 5.043188. The process exited successfully in 25m45s.
+        The shorter qwen35_vl_27b_pretrain_mock_config alias exists in its
+        family module but is not exported from the recipe package root, so
+        launcher lookup required the fully exported stock recipe named in the
+        command.
+
+  sft:
+    GB200:
+      status: verified
+      precision: bf16
+      enabled_features: {}
+      command: >
+        ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
+        --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft
+        --dataset medpix --deterministic
+        --pretrained_checkpoint
+        work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+        --max_steps 100 --warmup_iters 10 --save_interval 50
+        'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}'
+        'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}'
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        dataset.do_validation=false dataset.pad_to_max_length=true
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        model.recompute_granularity=full model.recompute_method=uniform
+        model.recompute_num_layers=1 model.recompute_modules=null
+        scheduler.lr_decay_iters=100 validation.eval_iters=0
+        validation.eval_interval=0 ddp.check_for_large_grads=true
+        --save_dir work/model-verification/qwen3.8-27b/sft-recompute/checkpoints
+        logger.log_interval=1 logger.log_throughput=true
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-recompute/resolved-config.yaml
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 2.116404
+        final_loss: 1.429567
+        last_10_steps_step_time_ms_avg: 24160.490
+        last_10_steps_model_tflops_per_gpu_avg: 58.010
+      expected_result: >
+        The 16-GB200 run completed exactly 100 full-SFT MedPix steps at
+        TP4/PP4, MBS4/GBS32, deterministic mode, and one-layer full-uniform
+        activation recompute. Loss remained finite from 2.116404 to 1.429567
+        with zero skipped or NaN iterations. Steps 91-100 averaged 24,160.490
+        ms and 58.010 model TFLOP/s/GPU. Step-50 and step-100 saves each
+        contain 20 files totaling 404,422,605,133 bytes, the tracker selects
+        iteration 100, and the process exited successfully in 45m42s. The
+        unchanged stock TP4/PP4 recipe without activation recompute reached the
+        training loop but OOMed near the approximately 184.3-GiB per-GPU
+        device limit; the successful command records the required config-only
+        recompute overrides.
+
+  sft_export_inference:
+    GB200:
+      status: verified
+      precision: bf16
+      depends_on: sft
+      commands:
+        - >
+          ./scripts/conversion/convert.sh export --executor slurm --device cpu
+          --nodes 1 --gpus-per-node 1 --mem 512G
+          --hf-model Qwen/Qwen3.8-27B
+          --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0
+          --megatron-path
+          work/model-verification/qwen3.8-27b/sft-recompute/checkpoints/iter_0000100
+          --hf-path work/model-verification/qwen3.8-27b/sft-recompute/hf-export
+        - >
+          ./scripts/inference/infer.sh --nodes 1 --gpus-per-node 1
+          --task hf-inference
+          --hf-model work/model-verification/qwen3.8-27b/sft-recompute/hf-export
+          --prompt "What was observed in the imaging?"
+          --image work/data/medpix/verification.png
+          --max-new-tokens 32 --chat-template --disable-thinking
+      last_verified: 2026-08-14
+      expected_result: >
+        CPU export from the exact step-100 SFT checkpoint completed all 956
+        mappings in 2m39s and reported that every source tensor was written.
+        The export contains 26 nonempty files totaling 55,583,128,197 bytes;
+        its index maps 1,199 keys to 18 safetensors shards with
+        55,562,855,904 serialized BF16 tensor bytes. The unchanged strict HF
+        verifier then reloaded the persisted export without any missing,
+        unexpected, mismatched, or error entries on one GB200 and generated
+        exactly 32 tokens at the 32-token maximum. The literal completion was:
+        "The imaging shows •  CT:  Hyperdense lesion in the right cerebellum.\n•
+        MRI:  Well-circumscribed, homogenous". Because deterministic mode is
+        persisted in the training checkpoint, Transformer Engine requires
+        nondeterministic algorithms to be explicitly disabled while
+        constructing the Megatron model for export. An initial strict attempt
+        without that environment
+        setting failed before conversion and deleted its partial output. The
+        successful retry explicitly disabled nondeterministic Transformer
+        Engine algorithms through runtime wiring outside this public card and
+        made no code change.
+
+  sft_long_context:
+    GB200:
+      status: verified
+      precision: bf16
+      enabled_features:
+        sequence_packing: in_batch
+        context_parallel_size: 2
+      command: >
+        ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
+        --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft
+        --dataset medpix
+        --pretrained_checkpoint
+        work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+        --max_steps 20 --warmup_iters 2 --seq_length 8192
+        --pipeline_model_parallel_size 2 --context_parallel_size 2
+        'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}'
+        'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}'
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        dataset.do_validation=false dataset.enable_in_batch_packing=true
+        dataset.defer_in_batch_packing_to_step=true
+        dataset.in_batch_packing_pad_to_multiple_of=4
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        model.calculate_per_token_loss=true model.recompute_granularity=full
+        model.recompute_method=uniform model.recompute_num_layers=1
+        ddp.average_in_collective=false scheduler.lr_decay_iters=20
+        checkpoint.load=null checkpoint.save=null validation.eval_iters=0
+        validation.eval_interval=0 ddp.check_for_large_grads=true
+        logger.log_interval=1 logger.log_throughput=true
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-long/resolved-config.yaml
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 2.343622
+        final_loss: 1.378064
+        last_10_steps_step_time_ms_avg: 8164.790
+        last_10_steps_model_tflops_per_gpu_avg: 362.530
+      expected_result: >
+        The 16-GB200 run completed exactly 20 full-SFT steps at
+        TP4/PP2/CP2, MBS4/GBS32, 8192 tokens, deferred processor-native
+        in-batch packing with pad-to-multiple-of-4, per-token loss, and
+        one-layer full-uniform activation recompute. Loss remained finite from
+        2.343622 to 1.378064 with zero skipped or NaN iterations. Steps 11-20
+        averaged 8,164.790 ms and 362.530 model TFLOP/s/GPU. The process exited
+        successfully without checkpoint output. An earlier otherwise identical
+        attempt with --deterministic reached the training loop but triggered the
+        explicit guard that packed sequence does not support deterministic mode;
+        the successful command omits that incompatible flag, as does the Qwen
+        3.5 MoE reference card.
+
+  peft:
+    GB200:
+      status: verified
+      precision: bf16
+      enabled_features: {}
+      command: >
+        ./scripts/training/train.sh --nodes 1 --gpus-per-node 4
+        --recipe qwen35_vl_27b_peft_2gpu_h100_bf16_config --mode lora
+        --tensor_model_parallel_size 4 --dataset medpix --deterministic
+        --pretrained_checkpoint
+        work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+        --max_steps 100 --warmup_iters 10 --save_interval 50
+        'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}'
+        'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}'
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        dataset.do_validation=false dataset.pad_to_max_length=true
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        model.recompute_granularity=full model.recompute_method=uniform
+        model.recompute_num_layers=1 model.recompute_modules=null
+        scheduler.lr_decay_iters=100 validation.eval_iters=0
+        validation.eval_interval=0 ddp.check_for_large_grads=true
+        --save_dir work/model-verification/qwen3.8-27b/peft-tp4-recompute/checkpoints
+        logger.log_interval=1 logger.log_throughput=true
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/peft-tp4-recompute/resolved-config.yaml
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 2.115661
+        final_loss: 1.558339
+        last_10_steps_step_time_ms_avg: 74048.550
+        last_10_steps_model_tflops_per_gpu_avg: 75.790
+      expected_result: >
+        The four-GB200 run completed exactly 100 deterministic MedPix LoRA
+        steps at TP4/PP1, MBS4/GBS32, and one-layer full-uniform activation
+        recompute. Loss remained finite from 2.115661 to 1.558339 with zero
+        skipped or NaN iterations. Steps 91-100 averaged 74,048.550 ms and
+        75.790 model TFLOP/s/GPU. Step-50 and step-100 adapter checkpoints each
+        contain 8 files totaling 2,294,105,245 bytes, the tracker selects
+        iteration 100, and the process exited successfully in 2h01m43s. The
+        unchanged stock TP2 two-GPU recipe without activation recompute OOMed;
+        a TP4 retry without recompute also OOMed near the approximately
+        184.3-GiB per-GPU device limit. The successful command records only the
+        required topology and recompute config overrides, with no code change.
+
+  checkpoint_resume:
+    GB200:
+      status: verified
+      precision: bf16
+      depends_on: pretrain
+      command: >
+        ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
+        --recipe qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config
+        --mode pretrain --deterministic
+        --load_dir work/model-verification/qwen3.8-27b/pretrain/checkpoints
+        --max_steps 20 --warmup_iters 2
+        --save_dir work/model-verification/qwen3.8-27b/pretrain-resume/checkpoints
+        --save_interval 10
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        checkpoint.ckpt_step=10 validation.eval_iters=0 validation.eval_interval=0
+        ddp.check_for_large_grads=true logger.log_interval=1
+        logger.log_throughput=true
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain-resume/resolved-config.yaml
+        scheduler.lr_decay_iters=20
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 5.452590
+        final_loss: 5.000320
+        last_10_steps_step_time_ms_avg: 45121.610
+        last_10_steps_model_tflops_per_gpu_avg: 35.050
+      resume_comparison:
+        reference_item: pretrain
+        sentinel_steps: [11, 20]
+        loss_relative_tolerance: 1.0e-2
+        loss_absolute_tolerance: 1.0e-6
+        sentinels_match: true
+      expected_result: >
+        The 16-GB200 run loaded the full iteration-10 distributed checkpoint,
+        including optimizer, scheduler, and RNG state, and explicitly started
+        training at iteration 10. Every reported loss from steps 11 through 20
+        exactly matched the uninterrupted reference at printed precision; the
+        maximum absolute difference was 0.0, including sentinel losses
+        5.452590 at step 11 and 5.000320 at step 20. The resumed steps averaged
+        45,121.610 ms and 35.050 model TFLOP/s/GPU, including first-step
+        compilation. The resulting iteration-20 checkpoint contains 20 files
+        totaling 56,110,835,024 bytes, its tracker records iteration 20, both
+        built-in evaluation passes completed at loss 5.043188, and the process
+        exited successfully in 17m27s.

From ab905ac72ccc9e45a081153655eda9ee469fdcff Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 14:09:21 -0700
Subject: [PATCH 02/10] fix(inference): forward multimodal token type IDs

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../compare_hf_and_megatron/compare.py        | 36 +++++++-
 .../unit_tests/test_compare_mask_handling.py  | 91 +++++++++++++++++--
 2 files changed, 115 insertions(+), 12 deletions(-)

diff --git a/examples/conversion/compare_hf_and_megatron/compare.py b/examples/conversion/compare_hf_and_megatron/compare.py
index 2e2efe6235..357dd8d5e0 100644
--- a/examples/conversion/compare_hf_and_megatron/compare.py
+++ b/examples/conversion/compare_hf_and_megatron/compare.py
@@ -279,6 +279,7 @@ def __init__(
         pixel_values=None,
         image_grid_thw=None,
         inference_context=None,
+        mm_token_type_ids=None,
     ):
         self.batch = dict(
             tokens=input_ids,
@@ -292,6 +293,8 @@ def __init__(
             self.batch["pixel_values"] = pixel_values
         if image_grid_thw is not None:
             self.batch["image_grid_thw"] = image_grid_thw
+        if mm_token_type_ids is not None:
+            self.batch["mm_token_type_ids"] = mm_token_type_ids
 
         self._yielded = False
 
@@ -332,6 +335,8 @@ def vlm_forward_step(data_iterator, model, **kwargs) -> torch.Tensor:
         forward_args["pixel_values"] = batch["pixel_values"]
     if "image_grid_thw" in batch:
         forward_args["image_grid_thw"] = batch["image_grid_thw"]
+    if "mm_token_type_ids" in batch:
+        forward_args["mm_token_type_ids"] = batch["mm_token_type_ids"]
 
     def loss_func(x, **kwargs):
         return x
@@ -441,7 +446,8 @@ def process_inputs(tokenizer, processor, image_path: Optional[str], prompt: str,
         tp_size: Tensor parallel size for padding sequence length
 
     Returns:
-        Tuple of (input_ids, pixel_values, image_grid_thw, token_type_ids)
+        Tuple of (input_ids, pixel_values, image_grid_thw, token_type_ids,
+        mm_token_type_ids)
     """
     if is_vl_model and image_path:
         messages = [
@@ -465,11 +471,15 @@ def process_inputs(tokenizer, processor, image_path: Optional[str], prompt: str,
         token_type_ids = inputs.get("token_type_ids")
         if token_type_ids is not None:
             token_type_ids = pad_input_ids_to_tp_multiple(token_type_ids, tp_size, 0)
+        mm_token_type_ids = inputs.get("mm_token_type_ids")
+        if mm_token_type_ids is not None:
+            mm_token_type_ids = pad_input_ids_to_tp_multiple(mm_token_type_ids, tp_size, 0)
         return (
             input_ids,
             inputs.get("pixel_values"),
             inputs.get("image_grid_thw"),
             token_type_ids,
+            mm_token_type_ids,
         )
     else:
         # Text-only processing for both VL models without images and regular LLMs
@@ -480,7 +490,7 @@ def process_inputs(tokenizer, processor, image_path: Optional[str], prompt: str,
             # Use tokenizer for regular LLMs
             inputs = tokenizer(prompt, return_tensors="pt")
         input_ids = pad_input_ids_to_tp_multiple(inputs.input_ids, tp_size, tokenizer.pad_token_id or 0)
-        return input_ids, None, None, None
+        return input_ids, None, None, None, None
 
 
 def _load_hf_model(args, is_vl_model: bool):
@@ -573,7 +583,16 @@ def _get_hf_forward_model(hf_model, pixel_values):
     return hf_model
 
 
-def _run_hf_inference(hf_model, input_ids, pixel_values, image_grid_thw, tokenizer, *, token_type_ids=None):
+def _run_hf_inference(
+    hf_model,
+    input_ids,
+    pixel_values,
+    image_grid_thw,
+    tokenizer,
+    *,
+    token_type_ids=None,
+    mm_token_type_ids=None,
+):
     """Run HuggingFace model inference and return results.
 
     Args:
@@ -582,7 +601,8 @@ def _run_hf_inference(hf_model, input_ids, pixel_values, image_grid_thw, tokeniz
         pixel_values: Pixel values for vision models (optional).
         image_grid_thw: Image grid dimensions (optional).
         tokenizer: Tokenizer for decoding.
-        token_type_ids: Multimodal token type IDs (optional).
+        token_type_ids: Legacy multimodal token type IDs (optional).
+        mm_token_type_ids: Multimodal token type IDs used for M-RoPE (optional).
 
     Returns:
         Tuple of (hf_logits, hf_next_token, hf_logits_stats, hf_top5_info, logits_shape).
@@ -613,6 +633,8 @@ def _run_hf_inference(hf_model, input_ids, pixel_values, image_grid_thw, tokeniz
             hf_inputs["image_grid_thw"] = image_grid_thw.to(hf_device)
         if token_type_ids is not None:
             hf_inputs["token_type_ids"] = token_type_ids.to(hf_device)
+        if mm_token_type_ids is not None:
+            hf_inputs["mm_token_type_ids"] = mm_token_type_ids.to(hf_device)
 
         hf_output = hf_forward_model(**hf_inputs)
 
@@ -848,7 +870,7 @@ def compare_models_one_step(args) -> None:
 
     # Process inputs
     print_rank_0(f"Processing inputs - Prompt: '{args.prompt}', Image: {args.image_path}")
-    input_ids, pixel_values, image_grid_thw, token_type_ids = process_inputs(
+    input_ids, pixel_values, image_grid_thw, token_type_ids, mm_token_type_ids = process_inputs(
         tokenizer, processor, args.image_path, args.prompt, is_vl_model, args.tp
     )
 
@@ -860,6 +882,8 @@ def compare_models_one_step(args) -> None:
         image_grid_thw = image_grid_thw.cuda()
     if token_type_ids is not None:
         token_type_ids = token_type_ids.cuda()
+    if mm_token_type_ids is not None:
+        mm_token_type_ids = mm_token_type_ids.cuda()
 
     print_rank_0(f"Input shape: {input_ids.shape}")
     print_rank_0(f"Pixel values shape: {pixel_values.shape if pixel_values is not None else 'None'}")
@@ -877,6 +901,7 @@ def compare_models_one_step(args) -> None:
             image_grid_thw,
             tokenizer,
             token_type_ids=token_type_ids,
+            mm_token_type_ids=mm_token_type_ids,
         )
 
     del hf_model
@@ -922,6 +947,7 @@ def compare_models_one_step(args) -> None:
                 attention_mask,
                 pixel_values,
                 image_grid_thw,
+                mm_token_type_ids=mm_token_type_ids,
             )
             megatron_output = fwd_bwd_function(
                 forward_step_func=vlm_forward_step,
diff --git a/tests/unit_tests/test_compare_mask_handling.py b/tests/unit_tests/test_compare_mask_handling.py
index 7a34025aaa..02962f787a 100644
--- a/tests/unit_tests/test_compare_mask_handling.py
+++ b/tests/unit_tests/test_compare_mask_handling.py
@@ -120,19 +120,22 @@ def test_vlm_inputs_preserve_image_token_types_and_tp_padding(self):
         image = object()
 
         with patch.object(compare, "load_image", return_value=image):
-            actual_input_ids, actual_pixels, image_grid_thw, actual_token_type_ids = compare.process_inputs(
-                tokenizer,
-                processor,
-                "/tmp/example.png",
-                "Describe this image.",
-                is_vl_model=True,
-                tp_size=2,
+            actual_input_ids, actual_pixels, image_grid_thw, actual_token_type_ids, actual_mm_token_type_ids = (
+                compare.process_inputs(
+                    tokenizer,
+                    processor,
+                    "/tmp/example.png",
+                    "Describe this image.",
+                    is_vl_model=True,
+                    tp_size=2,
+                )
             )
 
         torch.testing.assert_close(actual_input_ids, torch.tensor([[1, 2, 3, 7]]))
         assert actual_pixels is pixel_values
         assert image_grid_thw is None
         torch.testing.assert_close(actual_token_type_ids, torch.tensor([[0, 1, 0, 0]]))
+        assert actual_mm_token_type_ids is None
         processor.apply_chat_template.assert_called_once_with(
             [
                 {
@@ -149,6 +152,39 @@ def test_vlm_inputs_preserve_image_token_types_and_tp_padding(self):
             return_tensors="pt",
         )
 
+    def test_vlm_inputs_preserve_mm_token_types_and_tp_padding(self):
+        """VLM preprocessing keeps M-RoPE token types under their processor key."""
+        input_ids = torch.tensor([[1, 2, 3]])
+        mm_token_type_ids = torch.tensor([[0, 1, 0]])
+        pixel_values = torch.randn(3, 8)
+        image_grid_thw = torch.tensor([[1, 2, 2]])
+        processor = MagicMock()
+        processor.apply_chat_template.return_value = {
+            "input_ids": input_ids,
+            "pixel_values": pixel_values,
+            "image_grid_thw": image_grid_thw,
+            "mm_token_type_ids": mm_token_type_ids,
+        }
+        tokenizer = SimpleNamespace(pad_token_id=7)
+
+        with patch.object(compare, "load_image", return_value=object()):
+            actual_input_ids, actual_pixels, actual_grid, token_type_ids, actual_mm_token_type_ids = (
+                compare.process_inputs(
+                    tokenizer,
+                    processor,
+                    "/tmp/example.png",
+                    "Describe this image.",
+                    is_vl_model=True,
+                    tp_size=2,
+                )
+            )
+
+        torch.testing.assert_close(actual_input_ids, torch.tensor([[1, 2, 3, 7]]))
+        assert actual_pixels is pixel_values
+        assert actual_grid is image_grid_thw
+        assert token_type_ids is None
+        torch.testing.assert_close(actual_mm_token_type_ids, torch.tensor([[0, 1, 0, 0]]))
+
     def test_single_batch_iterator_stores_none_attention_mask(self):
         """Test that SingleBatchIterator preserves None attention_mask in batch dict."""
         input_ids = torch.tensor([[1, 2, 3]])
@@ -180,6 +216,21 @@ def test_vlm_forward_step_passes_none_attention_mask(self):
         assert "inference_context" not in call_kwargs
         assert "runtime_gather_output" not in call_kwargs
 
+    def test_vlm_forward_step_passes_mm_token_type_ids(self):
+        """Megatron receives processor-produced M-RoPE token types unchanged."""
+        mm_token_type_ids = torch.tensor([[0, 1, 0]])
+        iterator = SingleBatchIterator(
+            torch.tensor([[1, 2, 3]]),
+            torch.arange(3).unsqueeze(0),
+            None,
+            mm_token_type_ids=mm_token_type_ids,
+        )
+        mock_model = MagicMock(return_value=torch.randn(1, 3, 100))
+
+        vlm_forward_step(iterator, mock_model)
+
+        assert mock_model.call_args.kwargs["mm_token_type_ids"] is mm_token_type_ids
+
     def test_text_inference_forward_step_passes_static_context(self):
         """Test that the text path receives the cache context and gathered-logit request."""
         inference_context = object()
@@ -305,6 +356,32 @@ def test_hf_path_receives_multimodal_token_type_ids(self):
 
         assert mock_hf_model.call_args.kwargs["token_type_ids"] is token_type_ids
 
+    def test_hf_path_receives_mm_token_type_ids(self):
+        """Qwen M-RoPE token types reach HF under the exact processor key."""
+        mock_hf_model = MagicMock()
+        mock_output = MagicMock()
+        mock_output.logits = torch.randn(1, 3, 100)
+        mock_hf_model.return_value = mock_output
+        input_ids = torch.tensor([[1, 2, 3]])
+        mm_token_type_ids = torch.tensor([[0, 1, 0]])
+        mock_tokenizer = MagicMock()
+        mock_tokenizer.decode.return_value = "test"
+
+        with (
+            patch.object(compare, "_is_rank_0", return_value=True),
+            patch.object(compare, "print_rank_0"),
+        ):
+            _run_hf_inference(
+                mock_hf_model,
+                input_ids,
+                pixel_values=torch.randn(3, 8),
+                image_grid_thw=torch.tensor([[1, 2, 2]]),
+                tokenizer=mock_tokenizer,
+                mm_token_type_ids=mm_token_type_ids,
+            )
+
+        assert mock_hf_model.call_args.kwargs["mm_token_type_ids"] is mm_token_type_ids
+
     def test_hf_broadcast_uses_model_output_vocab_size(self):
         """Test that non-rank-0 buffers use the HF logits size instead of tokenizer vocab size."""
         broadcast_shapes = []

From af47959a906c942a63933fbed28d7fdc8a75d8ae Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 14:24:41 -0700
Subject: [PATCH 03/10] docs(verification): verify Qwen 3.8 manual forward pass

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../qwen3.8-27b/card.yaml                     | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/examples/model_verification_cards/qwen3.8-27b/card.yaml b/examples/model_verification_cards/qwen3.8-27b/card.yaml
index 69109becb1..9d73dafc38 100644
--- a/examples/model_verification_cards/qwen3.8-27b/card.yaml
+++ b/examples/model_verification_cards/qwen3.8-27b/card.yaml
@@ -7,9 +7,8 @@ summary: >
   timing and throughput metrics are sanity checks, not optimized performance
   results. All workloads in this card target GB200 hardware. Conversion and
   deterministic inference and all required functional training gates are
-  verified. The unchanged model-comparison script is currently incompatible
-  with the Qwen 3.8 processor because it does not forward mm_token_type_ids.
-  No dense-27B GB200 performance or Megatron FSDP recipe is available.
+  verified. No dense-27B GB200 performance or Megatron FSDP recipe is
+  available.
 verification_index:
   model_level:
     verified:
@@ -17,8 +16,8 @@ verification_index:
       - hf_to_megatron_gpu
       - megatron_to_hf_cpu
       - megatron_to_hf_gpu
+      - manual_forward_pass
       - inference
-    unverified: [manual_forward_pass]
   training:
     GB200:
       verified: [pretrain, sft, sft_export_inference, sft_long_context, peft, checkpoint_resume]
@@ -112,8 +111,9 @@ items:
       the metadata field vision_config.dtype=bfloat16.
 
   manual_forward_pass:
-    status: unverified
+    status: verified
     precision: bf16
+    bridge_commit: ab905ac72ccc9e45a081153655eda9ee469fdcff  # pragma: allowlist secret
     command: >
       ./scripts/inference/infer.sh --nodes 1 --gpus-per-node 4
       --task model-comparison
@@ -124,15 +124,15 @@ items:
       work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
       --tp 4 --image_path work/data/medpix/verification.png
       --prompt "What was observed in the imaging?"
-    last_verified: null
+    last_verified: 2026-08-14
     expected_result: >
-      The unchanged command loaded the pinned HF weights and prepared a
-      128-token input plus a 256-by-1536 pixel tensor, but Transformers rejected
-      the forward before logits were produced: image_grid_thw was present while
-      the newly required mm_token_type_ids input was absent. The processor emits
-      that field, but the unchanged comparison script does not forward it.
-      Therefore no next-token, cosine-similarity, or absolute-logit-difference
-      claim is made. The input was MedPix-VQA revision
+      The pinned HF weights and imported TP4 checkpoint both produced
+      1-by-128-by-248320 logits from a 128-token input and a 256-by-1536 pixel
+      tensor. Both selected token 248045, the assistant-start special token.
+      Cosine similarity: 0.995276 (99.53%); the maximum and mean absolute logit
+      differences were 1.125000 and 0.192757. This passes the required
+      next-token match and 0.99 cosine-similarity gate. The input was MedPix-VQA
+      revision
       69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0, case MPX1095;
       its 256-by-256 image has SHA-256
       f5210fcff6dad525c80fcd0f462b274c9da4d634ccd4f1cc76578f1bfe13646d.

From b1d162e24a607fdaf45bddf3ef16cb973d546ff2 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 14:43:03 -0700
Subject: [PATCH 04/10] perf(recipe): tune Qwen 3.5 27B pretrain for GB200

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../bridge/recipes/qwen_vl/__init__.py        |  2 +
 .../bridge/recipes/qwen_vl/gb200/__init__.py  |  2 +
 .../bridge/recipes/qwen_vl/gb200/qwen35_vl.py | 80 +++++++++++++++++++
 .../recipes/qwen_vl/test_qwen35_vl_recipes.py | 51 ++++++++++++
 4 files changed, 135 insertions(+)

diff --git a/src/megatron/bridge/recipes/qwen_vl/__init__.py b/src/megatron/bridge/recipes/qwen_vl/__init__.py
index 1ab27b154b..74c5f30690 100644
--- a/src/megatron/bridge/recipes/qwen_vl/__init__.py
+++ b/src/megatron/bridge/recipes/qwen_vl/__init__.py
@@ -14,6 +14,7 @@
 
 # Qwen3.5/Qwen3.6-VL GB200 models
 from .gb200.qwen35_vl import (
+    qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config,
     qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config,
     qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config,
 )
@@ -73,6 +74,7 @@
 __all__ = [
     # Qwen3.5-VL pretrain configs
     "qwen35_vl_9b_pretrain_mock_config",
+    "qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config",
     "qwen35_vl_35b_a3b_pretrain_mock_config",
     "qwen35_vl_122b_a10b_pretrain_mock_config",
     "qwen35_vl_397b_a17b_pretrain_mock_config",
diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/__init__.py b/src/megatron/bridge/recipes/qwen_vl/gb200/__init__.py
index 9f1e7327a7..e703373657 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/__init__.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/__init__.py
@@ -13,12 +13,14 @@
 # limitations under the License.
 
 from megatron.bridge.recipes.qwen_vl.gb200.qwen35_vl import (
+    qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config,
     qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config,
     qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config,
 )
 
 
 __all__ = [
+    "qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config",
     "qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config",
     "qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config",
 ]
diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
index ec157307c6..a4a523db7e 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
@@ -16,7 +16,10 @@
 
 from __future__ import annotations
 
+import torch
+
 from megatron.bridge.recipes.qwen_vl.h100.qwen35_vl import (
+    qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config,
     qwen35_vl_35b_a3b_peft_4gpu_h100_bf16_config,
     qwen35_vl_35b_a3b_sft_16gpu_h100_bf16_config,
 )
@@ -27,6 +30,82 @@
 from megatron.bridge.utils.cuda_graph import set_cuda_graph_modules
 
 
+def qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config() -> ConfigContainer:
+    """Return Qwen3.5-VL 27B projector pretraining for sixteen GB200 GPUs.
+
+    This keeps the dense pretraining objective and trainable-parameter contract
+    from the H100 recipe while using the measured GB200 execution layout. In
+    particular, deterministic mode must remain opt-in: Qwen3.5's Gated DeltaNet
+    replaces its fused FLA kernels with a torch-native reference path when
+    deterministic mode is enabled.
+    """
+    cfg = qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config()
+
+    cfg.model.tensor_model_parallel_size = 4
+    cfg.model.pipeline_model_parallel_size = 2
+    cfg.model.pipeline_dtype = torch.bfloat16
+    cfg.model.virtual_pipeline_model_parallel_size = None
+    cfg.model.context_parallel_size = 2
+    cfg.model.sequence_parallel = False
+    cfg.model.calculate_per_token_loss = True
+
+    cfg.train.global_batch_size = 32
+    cfg.train.micro_batch_size = 4
+
+    cfg.model.recompute_granularity = "full"
+    cfg.model.recompute_method = "uniform"
+    cfg.model.recompute_num_layers = 1
+    cfg.model.recompute_modules = None
+
+    # Qwen-VL's multimodal inputs are not yet a safe full-iteration graph
+    # target, and its mRoPE path is incompatible with fused RoPE.
+    cfg.model.apply_rope_fusion = False
+    cfg.model.cuda_graph_impl = "none"
+    set_cuda_graph_modules(cfg.model, [])
+    cfg.model.use_te_rng_tracker = False
+    cfg.rng.te_rng_tracker = False
+
+    cfg.mixed_precision = get_mixed_precision_config(cfg.mixed_precision)
+    cfg.mixed_precision.grad_reduce_in_fp32 = False
+    cfg.ddp.grad_reduce_in_fp32 = False
+    cfg.ddp.overlap_grad_reduce = False
+    cfg.ddp.overlap_param_gather = False
+    cfg.optimizer.overlap_param_gather = False
+    cfg.optimizer.overlap_param_gather_with_optimizer_step = False
+
+    cfg.dataset.do_validation = False
+    cfg.dataset.pad_to_max_length = True
+    cfg.validation.eval_interval = 0
+    cfg.validation.eval_iters = 0
+    cfg.checkpoint.load = None
+    cfg.checkpoint.save = None
+    cfg.logger.log_interval = 1
+    cfg.logger.log_throughput = True
+    cfg.logger.tensorboard_dir = None
+    cfg.dist.distributed_timeout_minutes = 30
+
+    cfg.comm_overlap = CommOverlapConfig(
+        tp_comm_overlap=False,
+        overlap_grad_reduce=False,
+        overlap_param_gather=False,
+        overlap_param_gather_with_optimizer_step=False,
+        overlap_moe_expert_parallel_comm=False,
+        delay_wgrad_compute=False,
+    )
+
+    cfg.env_vars = {
+        **COMMON_RECIPE_ENV_VARS,
+        "CUDA_DEVICE_MAX_CONNECTIONS": 32,
+        "NVLINK_DOMAIN_SIZE": 72,
+        "USE_MNNVL": 1,
+        "NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
+        "NVTE_FWD_LAYERNORM_SM_MARGIN": 20,
+        "NVTE_NORM_BWD_USE_CUDNN": 1,
+        "NVTE_NORM_FWD_USE_CUDNN": 1,
+    }
+    return cfg
+
+
 def _apply_qwen35_vl_35b_a3b_gb200_functional_defaults(cfg: ConfigContainer) -> None:
     """Apply shared settings for bounded GB200 functional runs."""
     cfg.model.tensor_model_parallel_size = 1
@@ -133,6 +212,7 @@ def qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config() -> ConfigContaine
 
 
 __all__ = [
+    "qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config",
     "qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config",
     "qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config",
 ]
diff --git a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
index 039872baa8..e0ecfe8a36 100644
--- a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
+++ b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
@@ -104,6 +104,7 @@
 ]
 
 _QWEN35_VL_GB200_FUNCS = [
+    _qwen35_vl_gb200_module.qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config,
     _qwen35_vl_gb200_module.qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config,
     _qwen35_vl_gb200_module.qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config,
 ]
@@ -628,6 +629,56 @@ def test_qwen35_vl_35b_a3b_peft_16gpu_h100_defaults(monkeypatch: pytest.MonkeyPa
     assert cfg.model.vision_cuda_graph_scope == ["attn", "mlp"]
 
 
+def test_qwen35_vl_27b_gb200_pretrain_defaults(monkeypatch: pytest.MonkeyPatch):
+    """The dense GB200 pretrain recipe should retain its measured execution policy."""
+    patch_recipe_module_global(monkeypatch, _qwen35_vl_h100_module, "AutoBridge", _FakeAutoBridge)
+
+    cfg = _qwen35_vl_gb200_module.qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config()
+
+    _assert_basic_config(cfg)
+    assert cfg.model.tensor_model_parallel_size == 4
+    assert cfg.model.pipeline_model_parallel_size == 2
+    assert cfg.model.pipeline_dtype == torch.bfloat16
+    assert cfg.model.virtual_pipeline_model_parallel_size is None
+    assert cfg.model.context_parallel_size == 2
+    assert cfg.model.sequence_parallel is False
+    assert cfg.model.calculate_per_token_loss is True
+
+    assert cfg.model.freeze_language_model is True
+    assert cfg.model.freeze_vision_model is True
+    assert cfg.model.freeze_vision_projection is False
+    assert cfg.train.global_batch_size == 32
+    assert cfg.train.micro_batch_size == 4
+
+    assert cfg.model.recompute_granularity == "full"
+    assert cfg.model.recompute_method == "uniform"
+    assert cfg.model.recompute_num_layers == 1
+    assert cfg.model.recompute_modules is None
+    assert cfg.model.apply_rope_fusion is False
+    assert cfg.model.cuda_graph_impl == "none"
+    assert cuda_graph_module_names(cfg.model) == []
+
+    assert cfg.mixed_precision.grad_reduce_in_fp32 is False
+    assert cfg.ddp.grad_reduce_in_fp32 is False
+    assert cfg.ddp.overlap_grad_reduce is False
+    assert cfg.ddp.overlap_param_gather is False
+    assert cfg.comm_overlap.tp_comm_overlap is False
+    assert cfg.comm_overlap.overlap_grad_reduce is False
+    assert cfg.comm_overlap.overlap_param_gather is False
+
+    assert cfg.dataset.do_validation is False
+    assert cfg.dataset.pad_to_max_length is True
+    assert cfg.validation.eval_interval == 0
+    assert cfg.validation.eval_iters == 0
+    assert cfg.checkpoint.load is None
+    assert cfg.checkpoint.save is None
+    assert cfg.logger.log_interval == 1
+    assert cfg.logger.log_throughput is True
+    assert cfg.env_vars["CUDA_DEVICE_MAX_CONNECTIONS"] == 32
+    assert cfg.env_vars["NVTE_NORM_BWD_USE_CUDNN"] == 1
+    assert cfg.env_vars["NVTE_NORM_FWD_USE_CUDNN"] == 1
+
+
 @pytest.mark.parametrize(
     ("recipe_func", "expected_lr", "is_peft"),
     [

From 79e92ecb87735cf74381b43e74ff3409b03968d9 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 14:49:06 -0700
Subject: [PATCH 05/10] fix(recipe): disable collective averaging with CP

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py     | 1 +
 tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
index a4a523db7e..d760f42dab 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
@@ -68,6 +68,7 @@ def qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config() -> ConfigContainer:
     cfg.mixed_precision = get_mixed_precision_config(cfg.mixed_precision)
     cfg.mixed_precision.grad_reduce_in_fp32 = False
     cfg.ddp.grad_reduce_in_fp32 = False
+    cfg.ddp.average_in_collective = False
     cfg.ddp.overlap_grad_reduce = False
     cfg.ddp.overlap_param_gather = False
     cfg.optimizer.overlap_param_gather = False
diff --git a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
index e0ecfe8a36..a5c439d910 100644
--- a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
+++ b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
@@ -660,6 +660,7 @@ def test_qwen35_vl_27b_gb200_pretrain_defaults(monkeypatch: pytest.MonkeyPatch):
 
     assert cfg.mixed_precision.grad_reduce_in_fp32 is False
     assert cfg.ddp.grad_reduce_in_fp32 is False
+    assert cfg.ddp.average_in_collective is False
     assert cfg.ddp.overlap_grad_reduce is False
     assert cfg.ddp.overlap_param_gather is False
     assert cfg.comm_overlap.tp_comm_overlap is False

From 40c62c650e9a3c2461acf7752e69cc0e569c00ec Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 14:57:55 -0700
Subject: [PATCH 06/10] perf(recipe): disable Qwen pretrain evaluation

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py     | 2 ++
 tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
index d760f42dab..04348b163b 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
@@ -76,6 +76,8 @@ def qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config() -> ConfigContainer:
 
     cfg.dataset.do_validation = False
     cfg.dataset.pad_to_max_length = True
+    cfg.train.eval_interval = 0
+    cfg.train.eval_iters = 0
     cfg.validation.eval_interval = 0
     cfg.validation.eval_iters = 0
     cfg.checkpoint.load = None
diff --git a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
index a5c439d910..24f37ab9f7 100644
--- a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
+++ b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
@@ -669,6 +669,8 @@ def test_qwen35_vl_27b_gb200_pretrain_defaults(monkeypatch: pytest.MonkeyPatch):
 
     assert cfg.dataset.do_validation is False
     assert cfg.dataset.pad_to_max_length is True
+    assert cfg.train.eval_interval == 0
+    assert cfg.train.eval_iters == 0
     assert cfg.validation.eval_interval == 0
     assert cfg.validation.eval_iters == 0
     assert cfg.checkpoint.load is None

From bdab7530dde9196c24d580bd3bb6ee23c0e8323a Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 15:19:05 -0700
Subject: [PATCH 07/10] perf(recipe): remove Qwen GB200 activation recompute

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py   | 9 ++++++---
 .../unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py | 6 +++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
index 04348b163b..6b301a67ff 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
@@ -52,9 +52,12 @@ def qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config() -> ConfigContainer:
     cfg.train.global_batch_size = 32
     cfg.train.micro_batch_size = 4
 
-    cfg.model.recompute_granularity = "full"
-    cfg.model.recompute_method = "uniform"
-    cfg.model.recompute_num_layers = 1
+    # The projector-only workload fits comfortably without activation
+    # recompute on GB200. Recompute repeats the expensive Gated DeltaNet
+    # forward kernels and materially reduces sustained throughput.
+    cfg.model.recompute_granularity = None
+    cfg.model.recompute_method = None
+    cfg.model.recompute_num_layers = None
     cfg.model.recompute_modules = None
 
     # Qwen-VL's multimodal inputs are not yet a safe full-iteration graph
diff --git a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
index 24f37ab9f7..79982602fa 100644
--- a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
+++ b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
@@ -650,9 +650,9 @@ def test_qwen35_vl_27b_gb200_pretrain_defaults(monkeypatch: pytest.MonkeyPatch):
     assert cfg.train.global_batch_size == 32
     assert cfg.train.micro_batch_size == 4
 
-    assert cfg.model.recompute_granularity == "full"
-    assert cfg.model.recompute_method == "uniform"
-    assert cfg.model.recompute_num_layers == 1
+    assert cfg.model.recompute_granularity is None
+    assert cfg.model.recompute_method is None
+    assert cfg.model.recompute_num_layers is None
     assert cfg.model.recompute_modules is None
     assert cfg.model.apply_rope_fusion is False
     assert cfg.model.cuda_graph_impl == "none"

From 6b7602ef37d1823385f7a342c57a627af4e97198 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 15:33:18 -0700
Subject: [PATCH 08/10] docs(verification): record Qwen 3.8 GB200 throughput

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../qwen3.8-27b/card.yaml                     | 52 ++++++++++++++++---
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/examples/model_verification_cards/qwen3.8-27b/card.yaml b/examples/model_verification_cards/qwen3.8-27b/card.yaml
index 9d73dafc38..51024692e0 100644
--- a/examples/model_verification_cards/qwen3.8-27b/card.yaml
+++ b/examples/model_verification_cards/qwen3.8-27b/card.yaml
@@ -3,12 +3,12 @@
 
 title: qwen3_8_27b
 summary: >
-  Performance disclaimer: this model has not been performance-tuned; reported
-  timing and throughput metrics are sanity checks, not optimized performance
-  results. All workloads in this card target GB200 hardware. Conversion and
-  deterministic inference and all required functional training gates are
-  verified. No dense-27B GB200 performance or Megatron FSDP recipe is
-  available.
+  Performance scope: pretrain_performance.GB200 uses the tuned canonical
+  projector-only MockVLM recipe and records 451.720 model TFLOP/s/GPU on 16
+  GB200 GPUs; timing from all functional items remains sanity evidence. All
+  workloads in this card target GB200 hardware. Conversion, deterministic
+  inference, and all required functional training gates are verified. No
+  dense-27B Megatron FSDP recipe is available.
 verification_index:
   model_level:
     verified:
@@ -21,6 +21,8 @@ verification_index:
   training:
     GB200:
       verified: [pretrain, sft, sft_export_inference, sft_long_context, peft, checkpoint_resume]
+  performance:
+    GB200: verified
 model:
   hf_id: Qwen/Qwen3.8-27B
   hf_revision: 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0  # pragma: allowlist secret
@@ -428,3 +430,41 @@ items:
         totaling 56,110,835,024 bytes, its tracker records iteration 20, both
         built-in evaluation passes completed at loss 5.043188, and the process
         exited successfully in 17m27s.
+
+  pretrain_performance:
+    GB200:
+      status: verified
+      precision: bf16
+      bridge_commit: bdab7530dde9196c24d580bd3bb6ee23c0e8323a  # pragma: allowlist secret
+      command: >
+        ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
+        --additional-slurm-params segment=4
+        --recipe qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config
+        --mode pretrain --pretrained_checkpoint
+        work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000
+        --max_steps 20 --warmup_iters 2
+        dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export
+        tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export
+        model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044
+        logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain-performance/resolved-config.yaml
+        scheduler.lr_decay_iters=20
+      last_verified: 2026-08-14
+      metrics:
+        initial_loss: 6.853174
+        final_loss: 4.975722
+        last_10_steps_step_time_ms_avg: 3105.340
+        last_10_steps_model_tflops_per_gpu_avg: 451.720
+        peak_allocated_memory_gib: 42.244
+      expected_result: >
+        On 16 GB200 GPUs in one NVL72 domain, the exact canonical BF16
+        projector-only MockVLM recipe completes 20 steps at TP4/PP2/CP2,
+        sequence length 4096, and MBS4/GBS32. Language and vision weights are
+        frozen while the multimodal projector trains. The recipe keeps fused
+        Gated DeltaNet execution enabled, disables unnecessary activation
+        recompute, evaluation, and checkpoint output, and retains NaN and
+        large-gradient checks. Loss remains finite from 6.853174 to 4.975722
+        with zero skipped or NaN iterations. Excluding first-iteration kernel
+        compilation, steps 11-20 average 3,105.340 ms and 451.720 model
+        TFLOP/s/GPU; peak allocated memory is 42.244 GiB. The process exits
+        successfully. This mock projector-only result is throughput evidence,
+        not full-model convergence evidence.

From 7f9864c4712f401213ecd3c734a2bf627c306438 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 16:06:34 -0700
Subject: [PATCH 09/10] perf(recipe): use TP2 for Qwen GB200 pretraining

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../bridge/recipes/qwen_vl/gb200/qwen35_vl.py       | 13 +++++++------
 .../recipes/qwen_vl/test_qwen35_vl_recipes.py       |  8 ++++----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
index 6b301a67ff..af6cd34d05 100644
--- a/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
+++ b/src/megatron/bridge/recipes/qwen_vl/gb200/qwen35_vl.py
@@ -16,8 +16,6 @@
 
 from __future__ import annotations
 
-import torch
-
 from megatron.bridge.recipes.qwen_vl.h100.qwen35_vl import (
     qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config,
     qwen35_vl_35b_a3b_peft_4gpu_h100_bf16_config,
@@ -41,11 +39,14 @@ def qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config() -> ConfigContainer:
     """
     cfg = qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config()
 
-    cfg.model.tensor_model_parallel_size = 4
-    cfg.model.pipeline_model_parallel_size = 2
-    cfg.model.pipeline_dtype = torch.bfloat16
+    # TP2 leaves enough memory headroom for cold Gated DeltaNet autotuning
+    # while PP1 removes pipeline bubbles. With DP8 and MBS4, GBS32 executes as
+    # one microbatch per rank; TP1/MBS2 exhausts GB200 memory during autotuning.
+    cfg.model.tensor_model_parallel_size = 2
+    cfg.model.pipeline_model_parallel_size = 1
+    cfg.model.pipeline_dtype = None
     cfg.model.virtual_pipeline_model_parallel_size = None
-    cfg.model.context_parallel_size = 2
+    cfg.model.context_parallel_size = 1
     cfg.model.sequence_parallel = False
     cfg.model.calculate_per_token_loss = True
 
diff --git a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
index 79982602fa..c2fd1f97a2 100644
--- a/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
+++ b/tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py
@@ -636,11 +636,11 @@ def test_qwen35_vl_27b_gb200_pretrain_defaults(monkeypatch: pytest.MonkeyPatch):
     cfg = _qwen35_vl_gb200_module.qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config()
 
     _assert_basic_config(cfg)
-    assert cfg.model.tensor_model_parallel_size == 4
-    assert cfg.model.pipeline_model_parallel_size == 2
-    assert cfg.model.pipeline_dtype == torch.bfloat16
+    assert cfg.model.tensor_model_parallel_size == 2
+    assert cfg.model.pipeline_model_parallel_size == 1
+    assert cfg.model.pipeline_dtype is None
     assert cfg.model.virtual_pipeline_model_parallel_size is None
-    assert cfg.model.context_parallel_size == 2
+    assert cfg.model.context_parallel_size == 1
     assert cfg.model.sequence_parallel is False
     assert cfg.model.calculate_per_token_loss is True
 

From e2c2f1fd76ea9e91d5bb6f58b92a95b19b54cee3 Mon Sep 17 00:00:00 2001
From: Chen Cui <chcui@nvidia.com>
Date: Fri, 14 Aug 2026 16:13:45 -0700
Subject: [PATCH 10/10] docs(verification): record Qwen TP2 GB200 throughput

Signed-off-by: Chen Cui <chcui@nvidia.com>
---
 .../qwen3.8-27b/card.yaml                     | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/examples/model_verification_cards/qwen3.8-27b/card.yaml b/examples/model_verification_cards/qwen3.8-27b/card.yaml
index 51024692e0..2645eac83d 100644
--- a/examples/model_verification_cards/qwen3.8-27b/card.yaml
+++ b/examples/model_verification_cards/qwen3.8-27b/card.yaml
@@ -4,7 +4,7 @@
 title: qwen3_8_27b
 summary: >
   Performance scope: pretrain_performance.GB200 uses the tuned canonical
-  projector-only MockVLM recipe and records 451.720 model TFLOP/s/GPU on 16
+  projector-only MockVLM recipe and records 1,230.090 model TFLOP/s/GPU on 16
   GB200 GPUs; timing from all functional items remains sanity evidence. All
   workloads in this card target GB200 hardware. Conversion, deterministic
   inference, and all required functional training gates are verified. No
@@ -435,7 +435,7 @@ items:
     GB200:
       status: verified
       precision: bf16
-      bridge_commit: bdab7530dde9196c24d580bd3bb6ee23c0e8323a  # pragma: allowlist secret
+      bridge_commit: 7f9864c4712f401213ecd3c734a2bf627c306438  # pragma: allowlist secret
       command: >
         ./scripts/training/train.sh --nodes 4 --gpus-per-node 4
         --additional-slurm-params segment=4
@@ -450,21 +450,24 @@ items:
         scheduler.lr_decay_iters=20
       last_verified: 2026-08-14
       metrics:
-        initial_loss: 6.853174
-        final_loss: 4.975722
-        last_10_steps_step_time_ms_avg: 3105.340
-        last_10_steps_model_tflops_per_gpu_avg: 451.720
-        peak_allocated_memory_gib: 42.244
+        initial_loss: 6.857419
+        final_loss: 5.071018
+        last_10_steps_step_time_ms_avg: 1139.500
+        last_10_steps_model_tflops_per_gpu_avg: 1230.090
+        peak_allocated_memory_gib: 173.760
       expected_result: >
         On 16 GB200 GPUs in one NVL72 domain, the exact canonical BF16
-        projector-only MockVLM recipe completes 20 steps at TP4/PP2/CP2,
-        sequence length 4096, and MBS4/GBS32. Language and vision weights are
-        frozen while the multimodal projector trains. The recipe keeps fused
-        Gated DeltaNet execution enabled, disables unnecessary activation
-        recompute, evaluation, and checkpoint output, and retains NaN and
-        large-gradient checks. Loss remains finite from 6.853174 to 4.975722
-        with zero skipped or NaN iterations. Excluding first-iteration kernel
-        compilation, steps 11-20 average 3,105.340 ms and 451.720 model
-        TFLOP/s/GPU; peak allocated memory is 42.244 GiB. The process exits
-        successfully. This mock projector-only result is throughput evidence,
-        not full-model convergence evidence.
+        projector-only MockVLM recipe completes 20 steps at TP2/PP1/CP1,
+        sequence length 4096, and MBS4/GBS32, which is one microbatch per DP
+        rank. Language and vision weights are frozen while the multimodal
+        projector trains. The recipe keeps fused Gated DeltaNet execution
+        enabled, disables unnecessary activation recompute, evaluation, and
+        checkpoint output, and retains NaN loss and gradient checks. Loss
+        remains finite from 6.857419 to 5.071018 with zero skipped or NaN
+        iterations. Excluding first-iteration kernel compilation, steps 11-20
+        average 1,139.500 ms and 1,230.090 model TFLOP/s/GPU; peak allocated
+        memory is 173.760 GiB. The process exits successfully. TP1/PP1 with
+        MBS2 was rejected because cold Gated DeltaNet autotuning exhausted the
+        approximately 185-GiB device memory, while TP2 retains about 11 GiB of
+        allocated-memory headroom. This mock projector-only result is
+        throughput evidence, not full-model convergence evidence.