From 7cc502ed3ccdd36780c820a472cfcc6486235d89 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomala Date: Fri, 31 Jul 2026 06:27:52 -0700 Subject: [PATCH 1/3] docs: add ModelOpt FP8 Cosmos Framework inference Signed-off-by: Krzysztof Tomala --- .../run_with_cosmos_framework.ipynb | 77 +++++++++++++++++++ .../reasoner/run_with_cosmos_framework.ipynb | 63 +++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb b/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb index 6d692701..f18a9e0c 100644 --- a/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb +++ b/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb @@ -624,6 +624,83 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Nano FP8: Text to Image\n", + "\n", + "To use FP8, download the `fp8` revision and run Generator inference with the local checkpoint.\n", + "\n", + "### Create Payload" + ], + "id": "fe1b922e" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "t2i_fp8_payload, t2i_fp8_output, t2i_fp8_model = create_payload(\"t2i\", backend=\"pytorch_fp8\")" + ], + "execution_count": null, + "outputs": [], + "id": "73ff907c" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run" + ], + "id": "aaf87ed7" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "%%bash\n", + "set -euo pipefail\n", + "\n", + "cd \"$COSMOS3_REPO\"\n", + "FP8_CHECKPOINT=\"$COSMOS3_REPO/checkpoints/Cosmos3-Nano-FP8\"\n", + "uvx hf@latest download nvidia/Cosmos3-Nano \\\n", + " --revision fp8 \\\n", + " --local-dir \"$FP8_CHECKPOINT\"\n", + "\n", + "CUDA_VISIBLE_DEVICES=\"$CUDA_VISIBLE_DEVICES\" LD_LIBRARY_PATH= \\\n", + "\"$COSMOS3_UV_ENV/bin/torchrun\" \\\n", + " --nproc-per-node=\"$COSMOS3_NUM_GPUS\" \\\n", + " --master-addr=\"$COSMOS3_MASTER_ADDR\" \\\n", + " --master-port=\"$COSMOS3_TEXT_MASTER_PORT\" \\\n", + " -m cosmos_framework.scripts.inference \\\n", + " --parallelism-preset=throughput \\\n", + " -i \"$COSMOS3_PYTORCH_FP8_T2I_INPUT\" \\\n", + " -o \"$COSMOS3_PYTORCH_FP8_T2I_OUTPUT\" \\\n", + " --checkpoint-path \"$FP8_CHECKPOINT\" \\\n", + " --seed=0\n" + ], + "execution_count": null, + "outputs": [], + "id": "e0cbe358" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### View Results" + ], + "id": "03e5c4b5" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "view_run(t2i_fp8_output)" + ], + "execution_count": null, + "outputs": [], + "id": "8970833a" + }, { "cell_type": "markdown", "metadata": {}, diff --git a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb index 55527d42..2c8af321 100644 --- a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb +++ b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb @@ -639,6 +639,69 @@ " print(json.dumps(json.loads(benchmark_path.read_text()).get(\"average\", {}), indent=2))\n" ] }, + { + "cell_type": "markdown", + "id": "da7c0e0a", + "metadata": {}, + "source": [ + "### 7a. Optional: Run Nano FP8 Text Inference\n", + "\n", + "To use FP8, download the `fp8` revision and run inference with the local checkpoint.\n", + "\n", + "Expected output file:\n", + "\n", + "```text\n", + "packages/cosmos3/outputs/cookbooks/cosmos3/reasoner/nano/cosmos_framework_nano_fp8_text/nano_text/reasoner_text.txt\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "228e450b", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "set -euo pipefail\n", + "\n", + "cd \"$COSMOS3_REPO\"\n", + "FP8_CHECKPOINT=\"$COSMOS3_REPO/checkpoints/Cosmos3-Nano-FP8\"\n", + "uvx hf@latest download nvidia/Cosmos3-Nano \\\n", + " --revision fp8 \\\n", + " --local-dir \"$FP8_CHECKPOINT\"\n", + "\n", + "COSMOS_TRAINING=false CUDA_VISIBLE_DEVICES=\"$CUDA_VISIBLE_DEVICES\" \\\n", + "MASTER_ADDR=\"$COSMOS3_MASTER_ADDR\" MASTER_PORT=\"$COSMOS3_NANO_TEXT_MASTER_PORT\" RANK=0 WORLD_SIZE=1 LOCAL_RANK=0 \\\n", + ".venv/bin/python -m cosmos_framework.scripts.inference \\\n", + " --parallelism-preset=latency \\\n", + " -i \"$COSMOS3_INPUT_DIR/nano_text.json\" \\\n", + " -o \"$COSMOS3_OUTPUT_ROOT/cosmos_framework_nano_fp8_text\" \\\n", + " --checkpoint-path \"$FP8_CHECKPOINT\" \\\n", + " --seed=0 \\\n", + " --benchmark\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fdbcd0f", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os, json\n", + "\n", + "output_root = Path(os.environ[\"COSMOS3_OUTPUT_ROOT\"])\n", + "text_path = output_root / \"cosmos_framework_nano_fp8_text\" / \"nano_text\" / \"reasoner_text.txt\"\n", + "benchmark_path = output_root / \"cosmos_framework_nano_fp8_text\" / \"benchmark.json\"\n", + "\n", + "print(text_path)\n", + "print(text_path.read_text())\n", + "if benchmark_path.exists():\n", + " print(json.dumps(json.loads(benchmark_path.read_text()).get(\"average\", {}), indent=2))\n" + ] + }, { "cell_type": "markdown", "id": "5e7b2393", From 200cf526a0f236340ada70717bf75d59104fc0b7 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomala Date: Tue, 11 Aug 2026 03:06:22 -0700 Subject: [PATCH 2/3] docs: clarify FP8 checkpoint downloads --- .../generator/audiovisual/run_with_cosmos_framework.ipynb | 8 +++++--- .../cosmos3/reasoner/run_with_cosmos_framework.ipynb | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb b/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb index f18a9e0c..456b911a 100644 --- a/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb +++ b/cookbooks/cosmos3/generator/audiovisual/run_with_cosmos_framework.ipynb @@ -630,7 +630,7 @@ "source": [ "## Nano FP8: Text to Image\n", "\n", - "To use FP8, download the `fp8` revision and run Generator inference with the local checkpoint.\n", + "FP8 checkpoints are published in the same Hugging Face repositories as the standard checkpoints, under the `fp8` revision rather than in separate FP8 repositories. This example downloads `nvidia/Cosmos3-Nano` at revision `fp8`. To use another released FP8 checkpoint, change `FP8_MODEL_ID` and `FP8_CHECKPOINT` below; keep `FP8_REVISION=fp8`.\n", "\n", "### Create Payload" ], @@ -662,9 +662,11 @@ "set -euo pipefail\n", "\n", "cd \"$COSMOS3_REPO\"\n", + "FP8_MODEL_ID=\"nvidia/Cosmos3-Nano\"\n", + "FP8_REVISION=\"fp8\"\n", "FP8_CHECKPOINT=\"$COSMOS3_REPO/checkpoints/Cosmos3-Nano-FP8\"\n", - "uvx hf@latest download nvidia/Cosmos3-Nano \\\n", - " --revision fp8 \\\n", + "uvx hf@latest download \"$FP8_MODEL_ID\" \\\n", + " --revision \"$FP8_REVISION\" \\\n", " --local-dir \"$FP8_CHECKPOINT\"\n", "\n", "CUDA_VISIBLE_DEVICES=\"$CUDA_VISIBLE_DEVICES\" LD_LIBRARY_PATH= \\\n", diff --git a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb index 2c8af321..8ad99332 100644 --- a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb +++ b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb @@ -646,7 +646,7 @@ "source": [ "### 7a. Optional: Run Nano FP8 Text Inference\n", "\n", - "To use FP8, download the `fp8` revision and run inference with the local checkpoint.\n", + "FP8 checkpoints are published in the same Hugging Face repositories as the standard checkpoints, under the `fp8` revision rather than in separate FP8 repositories. This example downloads `nvidia/Cosmos3-Nano` at revision `fp8`. To use another released FP8 checkpoint, change `FP8_MODEL_ID` and `FP8_CHECKPOINT` below; keep `FP8_REVISION=fp8`.\n", "\n", "Expected output file:\n", "\n", @@ -666,9 +666,11 @@ "set -euo pipefail\n", "\n", "cd \"$COSMOS3_REPO\"\n", + "FP8_MODEL_ID=\"nvidia/Cosmos3-Nano\"\n", + "FP8_REVISION=\"fp8\"\n", "FP8_CHECKPOINT=\"$COSMOS3_REPO/checkpoints/Cosmos3-Nano-FP8\"\n", - "uvx hf@latest download nvidia/Cosmos3-Nano \\\n", - " --revision fp8 \\\n", + "uvx hf@latest download \"$FP8_MODEL_ID\" \\\n", + " --revision \"$FP8_REVISION\" \\\n", " --local-dir \"$FP8_CHECKPOINT\"\n", "\n", "COSMOS_TRAINING=false CUDA_VISIBLE_DEVICES=\"$CUDA_VISIBLE_DEVICES\" \\\n", From b1eff4dba62a053208968659ac41e7483cf96d16 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomala Date: Tue, 11 Aug 2026 05:57:13 -0700 Subject: [PATCH 3/3] docs: support FSDP in FP8 reasoner cookbook --- .../reasoner/run_with_cosmos_framework.ipynb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb index 8ad99332..e156c18d 100644 --- a/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb +++ b/cookbooks/cosmos3/reasoner/run_with_cosmos_framework.ipynb @@ -648,6 +648,8 @@ "\n", "FP8 checkpoints are published in the same Hugging Face repositories as the standard checkpoints, under the `fp8` revision rather than in separate FP8 repositories. This example downloads `nvidia/Cosmos3-Nano` at revision `fp8`. To use another released FP8 checkpoint, change `FP8_MODEL_ID` and `FP8_CHECKPOINT` below; keep `FP8_REVISION=fp8`.\n", "\n", + "Set `FP8_NUM_GPUS=1` to run without FSDP. To shard the model with FSDP, expose multiple GPUs in `CUDA_VISIBLE_DEVICES` and set `FP8_NUM_GPUS` to their count (for example, `CUDA_VISIBLE_DEVICES=0,1` and `FP8_NUM_GPUS=2`).\n", + "\n", "Expected output file:\n", "\n", "```text\n", @@ -669,14 +671,23 @@ "FP8_MODEL_ID=\"nvidia/Cosmos3-Nano\"\n", "FP8_REVISION=\"fp8\"\n", "FP8_CHECKPOINT=\"$COSMOS3_REPO/checkpoints/Cosmos3-Nano-FP8\"\n", + "FP8_NUM_GPUS=\"${FP8_NUM_GPUS:-1}\"\n", "uvx hf@latest download \"$FP8_MODEL_ID\" \\\n", " --revision \"$FP8_REVISION\" \\\n", " --local-dir \"$FP8_CHECKPOINT\"\n", "\n", "COSMOS_TRAINING=false CUDA_VISIBLE_DEVICES=\"$CUDA_VISIBLE_DEVICES\" \\\n", - "MASTER_ADDR=\"$COSMOS3_MASTER_ADDR\" MASTER_PORT=\"$COSMOS3_NANO_TEXT_MASTER_PORT\" RANK=0 WORLD_SIZE=1 LOCAL_RANK=0 \\\n", - ".venv/bin/python -m cosmos_framework.scripts.inference \\\n", - " --parallelism-preset=latency \\\n", + ".venv/bin/torchrun \\\n", + " --nproc-per-node=\"$FP8_NUM_GPUS\" \\\n", + " --master-addr=\"$COSMOS3_MASTER_ADDR\" \\\n", + " --master-port=\"$COSMOS3_NANO_TEXT_MASTER_PORT\" \\\n", + " -m cosmos_framework.scripts.inference \\\n", + " --parallelism-preset=throughput \\\n", + " --dp-shard-size=\"$FP8_NUM_GPUS\" \\\n", + " --dp-replicate-size=1 \\\n", + " --cp-size=1 \\\n", + " --cfgp-size=1 \\\n", + " --no-use-torch-compile \\\n", " -i \"$COSMOS3_INPUT_DIR/nano_text.json\" \\\n", " -o \"$COSMOS3_OUTPUT_ROOT/cosmos_framework_nano_fp8_text\" \\\n", " --checkpoint-path \"$FP8_CHECKPOINT\" \\\n",