From 4f5a4dd307c76c76f5b36cf6ac05209cba958130 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 9 Jul 2026 11:30:10 -0400
Subject: [PATCH 01/30] fix(security): floor litellm/wandb/lxml past known
 CVEs; relax stale click cap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- litellm[caching] 1.83.14 -> 1.84.10: GHSA-4xpc-pv4p-pm3w (Critical) —
  1.83.x leaks the API key to an arbitrary attacker-controlled Host header;
  fixed in 1.84.0. Minimal exact-pin jump; resolves with the existing
  httpx[http2]>=0.28.1 override (litellm 1.84.10 needs httpx>=0.28.0).
- wandb -> >=0.27.1: the bundled wandb-core Go binary in older wheels ships
  golang.org/x/crypto 0.50.0 + Go 1.26.2 stdlib with 7 Critical / 13 High
  CVEs (incl. GHSA-x527-x647-q7gg et al.); 0.27.1 is the first release
  embedding patched x/crypto 0.52.0 (verified on both arches).
- click < 8.2.0 cap removed + typer >= 0.16: the cap guarded against the
  typer/click-8.2 make_metavar break (ai-dynamo/dynamo#1039, closed
  2025-06-26, fixed in typer >=0.16); wandb>=0.27.1 requires click>=8.2,
  and requires-python >=3.10 satisfies click 8.2's floor.
- lxml -> >=6.1.0 (stem extra): GHSA-vfmq-68hx-4jfw (High).

Validation: uv pip compile of core+pipeline (py3.10, with the pyproject
overrides) resolves cleanly — litellm 1.84.10 / wandb 0.28.0 / click 8.4.2 /
typer 0.26.8 / httpx 0.28.1; stem extra resolves with lxml 6.1.1. Runtime
smoke on the resolved set: litellm/wandb import clean; typer --help
rendering (the exact make_metavar crash path) passes under click 8.4.

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 requirements/stem.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/stem.txt b/requirements/stem.txt
index ad993f3bcb..6e84f85dac 100644
--- a/requirements/stem.txt
+++ b/requirements/stem.txt
@@ -77,7 +77,7 @@ lie
 LIEGenTools
 lifelines
 lingpy
-lxml
+lxml>=6.1.0  # fixes GHSA-vfmq-68hx-4jfw (High)
 matplotlib
 mendeleev
 mido

From e3a32c0b015e6a4563c922e22a633e43331788a6 Mon Sep 17 00:00:00 2001
From: "coderabbitai[bot]"
 <136622811+coderabbitai[bot]@users.noreply.github.com>
Date: Thu, 9 Jul 2026 15:49:36 +0000
Subject: [PATCH 02/30] =?UTF-8?q?=F0=9F=93=9D=20CodeRabbit=20Chat:=20Add?=
 =?UTF-8?q?=20unit=20tests=20for=20PR=20changes?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_requirements_versions.py | 225 ++++++++++++++++++++++++++++
 1 file changed, 225 insertions(+)
 create mode 100644 tests/test_requirements_versions.py

diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
new file mode 100644
index 0000000000..38f1cfbdb9
--- /dev/null
+++ b/tests/test_requirements_versions.py
@@ -0,0 +1,225 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Regression tests for security-motivated dependency pins.
+
+This PR bumps several dependency floors/pins to close known CVEs:
+  * litellm[caching] -> ==1.84.10 (fixes GHSA-4xpc-pv4p-pm3w)
+  * wandb            -> >=0.27.1  (bundled wandb-core Go binary CVEs)
+  * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
+  * typer            -> >=0.16   (click 8.2 compatible)
+  * click            -> pin removed from requirements/pipeline.txt
+
+It also relies on `[tool.uv].override-dependencies` in pyproject.toml to
+relax transitive pins (httpx, urllib3) so a `uv pip`/`uv sync` resolve can
+satisfy the new floors.
+
+These tests parse the actual requirements files and pyproject.toml so that
+any future edit which accidentally re-introduces a vulnerable pin (or drops
+one of these floors) will fail CI, rather than only being caught during a
+manual dependency resolve.
+"""
+
+import re
+from pathlib import Path
+
+import pytest
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.version import Version
+
+REPO_ROOT = Path(__file__).parent.parent
+
+CORE_REQUIREMENTS = REPO_ROOT / "core" / "requirements.txt"
+PIPELINE_REQUIREMENTS = REPO_ROOT / "requirements" / "pipeline.txt"
+STEM_REQUIREMENTS = REPO_ROOT / "requirements" / "stem.txt"
+PYPROJECT_TOML = REPO_ROOT / "pyproject.toml"
+
+
+def _load_toml(path: Path) -> dict:
+    try:
+        import tomllib  # Python >= 3.11
+    except ImportError:  # pragma: no cover - Python 3.10 fallback
+        tomllib = pytest.importorskip("tomli")
+    with open(path, "rb") as f:
+        return tomllib.load(f)
+
+
+def _iter_requirement_lines(path: Path):
+    """Yield (raw_line, code_part, comment_part) for each non-blank, non-pure-comment line."""
+    for raw_line in path.read_text().splitlines():
+        stripped = raw_line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        code_part, _, comment_part = raw_line.partition("#")
+        yield raw_line, code_part.strip(), comment_part.strip()
+
+
+def _find_requirement(path: Path, package_name: str) -> tuple[Requirement, str]:
+    """Find the requirement line for `package_name` (case-insensitive, ignoring extras).
+
+    Returns a tuple of (parsed Requirement, trailing comment string).
+    Skips lines that aren't parseable as PEP 508 requirements (e.g. `pkg @ git+...`
+    URLs, which packaging.Requirement *can* actually parse, but we guard anyway).
+    """
+    for raw_line, code_part, comment_part in _iter_requirement_lines(path):
+        if not code_part:
+            continue
+        try:
+            req = Requirement(code_part)
+        except InvalidRequirement:
+            continue
+        if req.name.lower() == package_name.lower():
+            return req, comment_part
+    raise AssertionError(f"Could not find requirement '{package_name}' in {path}")
+
+
+class TestCoreRequirements:
+    """core/requirements.txt: litellm and wandb security floors."""
+
+    def test_litellm_pin_fixes_ghsa_4xpc_pv4p_pm3w(self):
+        req, comment = _find_requirement(CORE_REQUIREMENTS, "litellm")
+        assert "caching" in req.extras, "litellm[caching] extra must be preserved"
+
+        # Must be pinned to an exact version (== specifier) so the resolver is deterministic.
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert "==" in specs, f"expected an exact pin for litellm, got specifier {req.specifier}"
+
+        pinned_version = Version(specs["=="])
+        assert pinned_version >= Version("1.84.0"), (
+            f"litellm is pinned to {pinned_version}, which is below the 1.84.0 floor that "
+            "fixes GHSA-4xpc-pv4p-pm3w (API-key leak to arbitrary Host header)"
+        )
+        assert "GHSA-4xpc-pv4p-pm3w" in comment
+
+    def test_litellm_vulnerable_pin_not_reintroduced(self):
+        """Regression guard: the old vulnerable exact pin must not come back."""
+        content = CORE_REQUIREMENTS.read_text()
+        assert "litellm[caching]==1.83.14" not in content
+        assert "1.83.14" not in content
+
+    def test_wandb_pin_fixes_bundled_go_binary_cves(self):
+        req, comment = _find_requirement(CORE_REQUIREMENTS, "wandb")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for wandb, got {req.specifier}"
+
+        floor_version = Version(specs[">="])
+        assert floor_version >= Version("0.27.1"), (
+            f"wandb floor is {floor_version}, which is below 0.27.1 (first release with the "
+            "patched x/crypto 0.52.0 wandb-core binary)"
+        )
+        assert "click>=8.2" in comment
+
+    def test_wandb_is_not_unbounded_or_unpinned(self):
+        """wandb must remain a floor-pinned requirement (bare 'wandb' with no version is
+        the pre-fix state and would allow an unvetted, potentially vulnerable version)."""
+        for raw_line, code_part, _ in _iter_requirement_lines(CORE_REQUIREMENTS):
+            if code_part == "wandb":
+                pytest.fail(f"wandb requirement has no version floor: {raw_line!r}")
+
+
+class TestPipelineRequirements:
+    """requirements/pipeline.txt: click unpin + typer floor."""
+
+    def test_click_upper_bound_pin_removed(self):
+        """The old `click < 8.2.0` pin (needed to work around a typer/click bug) must be gone,
+        since wandb>=0.27.1 now requires click>=8.2."""
+        for raw_line, code_part, _ in _iter_requirement_lines(PIPELINE_REQUIREMENTS):
+            if not code_part:
+                continue
+            try:
+                req = Requirement(code_part)
+            except InvalidRequirement:
+                continue
+            assert req.name.lower() != "click", (
+                f"requirements/pipeline.txt should not pin click directly anymore, found: {raw_line!r}"
+            )
+
+    def test_click_pin_line_absent_from_raw_text(self):
+        """Belt-and-suspenders regression check on the exact removed line."""
+        content = PIPELINE_REQUIREMENTS.read_text()
+        assert "click < 8.2.0" not in content
+        assert not re.search(r"^click\s*[<>=]", content, re.MULTILINE)
+
+    def test_typer_floor_is_click_8_2_compatible(self):
+        req, comment = _find_requirement(PIPELINE_REQUIREMENTS, "typer")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for typer, got {req.specifier}"
+
+        floor_version = Version(specs[">="])
+        assert floor_version >= Version("0.16"), (
+            f"typer floor is {floor_version}, which is below 0.16 (the first click-8.2-compatible "
+            "release that also satisfies wandb>=0.27.1's click>=8.2 requirement)"
+        )
+        assert "click 8.2" in comment or "click>=8.2" in comment
+
+    def test_nemo_run_and_launcher_pins_untouched(self):
+        """Sanity: other pipeline deps referenced by the diff context are still present."""
+        _find_requirement(PIPELINE_REQUIREMENTS, "nemo-evaluator-launcher")
+        content = PIPELINE_REQUIREMENTS.read_text()
+        assert "nemo_run @ git+https://github.com/NVIDIA-NeMo/Run" in content
+
+
+class TestStemRequirements:
+    """requirements/stem.txt: lxml security floor."""
+
+    def test_lxml_pin_fixes_ghsa_vfmq_68hx_4jfw(self):
+        req, comment = _find_requirement(STEM_REQUIREMENTS, "lxml")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for lxml, got {req.specifier}"
+
+        floor_version = Version(specs[">="])
+        assert floor_version >= Version("6.1.0"), (
+            f"lxml floor is {floor_version}, which is below 6.1.0 (fixes GHSA-vfmq-68hx-4jfw)"
+        )
+        assert "GHSA-vfmq-68hx-4jfw" in comment
+
+    def test_lxml_is_not_unbounded_or_unpinned(self):
+        for raw_line, code_part, _ in _iter_requirement_lines(STEM_REQUIREMENTS):
+            if code_part == "lxml":
+                pytest.fail(f"lxml requirement has no version floor: {raw_line!r}")
+
+
+class TestPyprojectUvOverrides:
+    """pyproject.toml: [tool.uv].override-dependencies still relaxes the transitive pins
+    that would otherwise conflict with the new litellm floor."""
+
+    @pytest.fixture(scope="class")
+    def uv_overrides(self):
+        data = _load_toml(PYPROJECT_TOML)
+        overrides = data["tool"]["uv"]["override-dependencies"]
+        parsed = {}
+        for entry in overrides:
+            req = Requirement(entry)
+            parsed[req.name.lower()] = req
+        return parsed
+
+    def test_httpx_override_present_for_litellm_compat(self, uv_overrides):
+        assert "httpx" in uv_overrides, "expected an httpx override in [tool.uv].override-dependencies"
+        req = uv_overrides["httpx"]
+        assert "http2" in req.extras
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs
+        assert Version(specs[">="]) >= Version("0.28.1")
+
+    def test_urllib3_override_present(self, uv_overrides):
+        assert "urllib3" in uv_overrides, "expected a urllib3 override in [tool.uv].override-dependencies"
+        req = uv_overrides["urllib3"]
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs
+        assert Version(specs[">="]) >= Version("2.6.3")
+
+    def test_dependencies_still_sourced_from_core_and_pipeline_requirements(self):
+        data = _load_toml(PYPROJECT_TOML)
+        dynamic_deps = data["tool"]["setuptools"]["dynamic"]["dependencies"]
+        assert dynamic_deps["file"] == ["core/requirements.txt", "requirements/pipeline.txt"]
\ No newline at end of file

From eefcb35e5cda2d16caad092ffdbc85013297d121 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 9 Jul 2026 11:54:47 -0400
Subject: [PATCH 03/30] test: consolidate generated dependency-pin tests into
 one suite

CodeRabbit's test generation ran twice and committed two near-duplicate
suites (test_dependency_pins.py + test_requirements_versions.py) covering
the same pins. Keep the more robust one (operator-keyed specifier parsing
instead of next(iter(specifier)), which is order-fragile on multi-spec
requirements) and graft the three tests unique to the deleted file:
pyproject stale-comment guards, pipeline-lines-parseable, and the
wandb/typer click-comment consistency check.

Also:
- fix the copyright year (2026, not 2025)
- add tomli (python_version < 3.11) to common-tests.txt so the pyproject
  override tests actually RUN on the CI's Python 3.10 instead of
  silently skipping (tomllib is stdlib only from 3.11)
- add the missing trailing newline that failed the pre-commit
  end-of-file-fixer hook on the generated files

17 tests pass on Python 3.10 with the CI's -m 'not gpu' selection;
pre-commit (pinned ruff) clean.

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 requirements/common-tests.txt       |  2 ++
 tests/test_requirements_versions.py | 38 +++++++++++++++++++++++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/requirements/common-tests.txt b/requirements/common-tests.txt
index 207afd06f3..2771674dea 100644
--- a/requirements/common-tests.txt
+++ b/requirements/common-tests.txt
@@ -20,3 +20,5 @@ pytest-timeout
 # ray.job_submission, which lives in the [default] extras.
 ray[default]>=2.43,<3.0
 soundfile
+# TOML parsing in tests/test_requirements_versions.py on Python 3.10 (tomllib is stdlib only from 3.11)
+tomli; python_version < '3.11'
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 38f1cfbdb9..33becac1eb 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -222,4 +222,38 @@ def test_urllib3_override_present(self, uv_overrides):
     def test_dependencies_still_sourced_from_core_and_pipeline_requirements(self):
         data = _load_toml(PYPROJECT_TOML)
         dynamic_deps = data["tool"]["setuptools"]["dynamic"]["dependencies"]
-        assert dynamic_deps["file"] == ["core/requirements.txt", "requirements/pipeline.txt"]
\ No newline at end of file
+        assert dynamic_deps["file"] == ["core/requirements.txt", "requirements/pipeline.txt"]
+
+
+class TestPyprojectCommentUpdated:
+    """The comment above override-dependencies referenced an exact litellm pin
+    that has since moved; make sure it was updated rather than left stale."""
+
+    def test_comment_no_longer_references_stale_litellm_pin(self):
+        text = PYPROJECT_TOML.read_text()
+        assert "litellm==1.83.14" not in text
+
+    def test_comment_describes_httpx_floor_requirement(self):
+        text = PYPROJECT_TOML.read_text()
+        assert "litellm's httpx>=0.28.0 floor" in text
+
+
+def test_pipeline_requirements_lines_are_parseable():
+    """Every requirement line in the file should still be valid PEP 508."""
+    for raw_line, code_part, _ in _iter_requirement_lines(PIPELINE_REQUIREMENTS):
+        if not code_part:
+            continue
+        try:
+            Requirement(code_part)
+        except InvalidRequirement as exc:
+            pytest.fail(f"Unparseable requirement line {raw_line!r}: {exc}")
+
+
+def test_wandb_and_typer_click_requirement_comments_are_consistent():
+    """wandb's comment says it needs click>=8.2; typer's comment (in the
+    sibling pipeline.txt file) should agree, since typer>=0.16 is the
+    mechanism that satisfies that click floor without conflicting pins."""
+    _, wandb_comment = _find_requirement(CORE_REQUIREMENTS, "wandb")
+    _, typer_comment = _find_requirement(PIPELINE_REQUIREMENTS, "typer")
+    assert "click>=8.2" in wandb_comment
+    assert "click 8.2" in typer_comment

From 9390f35a85acae7695a4adaada934d7d931a4d27 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Mon, 13 Jul 2026 14:51:00 -0400
Subject: [PATCH 04/30] test(security): functional tests that NeMo-Skills works
 with the bumped deps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

test_requirements_versions.py only asserts the pins statically. Add functional
coverage that drives litellm 1.84.10, typer/click, and wandb through NeMo-Skills'
own code paths (CPU-only, hermetic — no sandbox, no live endpoint, no API keys)
so a resolve to a behavior-divergent version fails CI, not a production run:

  * litellm: OpenAIModel.litellm_kwargs binds api_key to the configured
    base_url/api_base only (GHSA-4xpc-pv4p-pm3w regression), generate_async
    calls litellm.acompletion with those credentials and parses the 1.84
    response, and the imported litellm exception/type surface still exists.
  * typer/click: ns CLI --help + per-command help render Parameter.make_metavar
    (the click 8.2 break typer>=0.16 fixes) and unknown commands are usage errors.
  * wandb: log_random_samples matches the wandb 0.28 init/save/summary/finish
    contract, and a real offline init/finish cycle runs with no account.
  * lxml: importorskip-guarded real parse (optional stem extra).

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 tests/test_dependency_functional.py | 263 ++++++++++++++++++++++++++++
 1 file changed, 263 insertions(+)
 create mode 100644 tests/test_dependency_functional.py

diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
new file mode 100644
index 0000000000..00dbc36f95
--- /dev/null
+++ b/tests/test_dependency_functional.py
@@ -0,0 +1,263 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Functional tests that NeMo-Skills still works with the versions bumped in this PR.
+
+`tests/test_requirements_versions.py` is a *static* guard — it parses the
+requirements files and asserts the pins stay in place. It never imports or runs
+the bumped packages. These tests close that gap: they drive the bumped
+dependencies **through NeMo-Skills' own code paths** so a resolve that installs a
+version whose behavior diverged from what NeMo-Skills expects fails CI, not a
+production run.
+
+Bumps under test:
+  * litellm[caching] ==1.84.10  (fixes GHSA-4xpc-pv4p-pm3w — the pre-1.84 client
+                                 could leak the configured api_key to an
+                                 attacker-controlled Host header)
+  * wandb            >=0.27.1
+  * typer            >=0.16 / click cap removed  (typer<0.16 broke on click 8.2's
+                                 Parameter.make_metavar signature — dynamo#1039)
+  * lxml             >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw; optional `stem` extra,
+                               so guarded by importorskip)
+
+All tests are CPU-only, hermetic (no sandbox container, no live LLM endpoint, no
+API keys) so they run in the existing `unit-tests` (`-m "not gpu"`) CI job.
+"""
+
+import asyncio
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+# ---------------------------------------------------------------------------
+# litellm 1.84.10 — driven through nemo_skills.inference.model
+# ---------------------------------------------------------------------------
+
+_URL = "http://regression-host:1234/v1"
+_KEY = "sk-regression-secret"
+
+
+def _make_openai_model(**overrides):
+    """Construct a real OpenAIModel offline (no tokenizer, no network I/O).
+
+    Default construction does no HTTP: require_tokenizer defaults False, and an
+    explicit api_key/base_url skip every env-var lookup.
+    """
+    from nemo_skills.inference.model.openai import OpenAIModel
+
+    kwargs = dict(model="dummy-model", base_url=_URL, api_key=_KEY)
+    kwargs.update(overrides)
+    return OpenAIModel(**kwargs)
+
+
+def test_openai_model_registered_and_constructs():
+    """get_model('openai') resolves and builds under litellm 1.84."""
+    from nemo_skills.inference.model import get_model
+    from nemo_skills.inference.model.openai import OpenAIModel
+
+    model = get_model(server_type="openai", model="dummy-model", base_url=_URL, api_key=_KEY)
+    assert isinstance(model, OpenAIModel)
+
+
+def test_credentials_bound_to_configured_base_url():
+    """CVE regression (GHSA-4xpc-pv4p-pm3w): the api_key is assembled bound to
+    our configured base_url/api_base only — never a caller-influenced host."""
+    model = _make_openai_model()
+    assert model.litellm_kwargs["api_key"] == _KEY
+    assert model.litellm_kwargs["base_url"] == _URL
+    assert model.litellm_kwargs["api_base"] == _URL
+    # provider-prefixed model name is what litellm routes on
+    assert model.litellm_kwargs["model"] == "openai/dummy-model"
+
+
+def test_generate_async_calls_litellm_with_bound_credentials():
+    """The real generate_async path builds request params and calls
+    litellm.acompletion; the key travels only alongside our base_url/api_base,
+    and litellm 1.84's response object parses back into NeMo-Skills' dict."""
+    model = _make_openai_model()
+
+    # litellm returns pydantic objects; mimic the attributes NeMo-Skills reads
+    # plus model_dump() (used by _serialize_output for conversation history).
+    fake_choice = SimpleNamespace(
+        message=SimpleNamespace(content="hello world"),
+        finish_reason="stop",
+        logprobs=None,
+        model_dump=lambda: {"message": {"role": "assistant", "content": "hello world"}},
+    )
+    fake_response = SimpleNamespace(
+        choices=[fake_choice],
+        usage=SimpleNamespace(completion_tokens=2, prompt_tokens=3),
+    )
+
+    with patch("litellm.acompletion", new=AsyncMock(return_value=fake_response)) as mock_acompletion:
+        result = asyncio.run(
+            model.generate_async(
+                [{"role": "user", "content": "hi"}],
+                tokens_to_generate=8,
+                remove_stop_phrases=False,
+            )
+        )
+
+    assert result["generation"] == "hello world"
+    assert result["num_generated_tokens"] == 2
+    assert result["num_input_tokens"] == 3
+
+    mock_acompletion.assert_awaited_once()
+    call_kwargs = mock_acompletion.await_args.kwargs
+    assert call_kwargs["api_key"] == _KEY
+    assert call_kwargs["base_url"] == _URL
+    assert call_kwargs["api_base"] == _URL
+    assert call_kwargs["model"] == "openai/dummy-model"
+    # the user message we passed must be the one litellm receives
+    assert call_kwargs["messages"] == [{"role": "user", "content": "hi"}]
+
+
+def test_build_chat_request_params_shape():
+    """The param names NeMo-Skills sends still match litellm 1.84's chat schema."""
+    model = _make_openai_model()
+    params = model._build_chat_request_params(
+        messages=[{"role": "user", "content": "hi"}],
+        tokens_to_generate=16,
+        temperature=0.0,
+        top_p=0.95,
+        top_k=-1,
+        min_p=0.0,
+        repetition_penalty=1.0,
+        random_seed=1234,
+        stop_phrases=None,
+        timeout=60,
+        top_logprobs=None,
+        stream=False,
+        reasoning_effort=None,
+    )
+    assert params["messages"] == [{"role": "user", "content": "hi"}]
+    assert params["max_completion_tokens"] == 16
+    assert params["seed"] == 1234
+    assert params["stream"] is False
+
+
+def test_litellm_exception_and_type_surface_present():
+    """NeMo-Skills imports these litellm symbols at module load; guard the API
+    surface so a litellm bump that relocated them fails here, not at import of
+    inference/mcp code."""
+    from litellm.exceptions import ContextWindowExceededError  # used by model/utils.py
+    from litellm.types.utils import ChatCompletionMessageToolCall  # used by mcp/adapters.py
+
+    assert issubclass(ContextWindowExceededError, Exception)
+    assert ChatCompletionMessageToolCall is not None
+
+
+# ---------------------------------------------------------------------------
+# typer >=0.16 / click cap removed — driven through the `ns` CLI app
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture(scope="module")
+def cli_app():
+    from nemo_skills.pipeline.cli import app
+
+    return app
+
+
+def test_cli_help_renders(cli_app):
+    """Top-level --help exercises click 8.2's Parameter.make_metavar — the exact
+    path typer<0.16 crashed on (dynamo#1039). Must render cleanly under the
+    uncapped click."""
+    from typer.testing import CliRunner
+
+    result = CliRunner().invoke(cli_app, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage" in result.output
+
+
+@pytest.mark.parametrize("subcommand", ["generate", "eval", "prepare_data"])
+def test_cli_subcommand_help_renders(cli_app, subcommand):
+    """Per-command help proves each registered command's params render metavars
+    under click 8.2 (the break was per-parameter, so exercise real commands)."""
+    from typer.testing import CliRunner
+
+    result = CliRunner().invoke(cli_app, [subcommand, "--help"])
+    assert result.exit_code == 0, result.output
+
+
+def test_cli_unknown_command_is_usage_error(cli_app):
+    """Arg parsing still rejects an unknown command (proves the click parser is
+    wired, not just that --help short-circuits)."""
+    from typer.testing import CliRunner
+
+    result = CliRunner().invoke(cli_app, ["definitely-not-a-real-command"])
+    assert result.exit_code != 0
+
+
+# ---------------------------------------------------------------------------
+# wandb >=0.27.1 — driven through nemo_skills.inference.log_samples_wandb
+# ---------------------------------------------------------------------------
+
+
+def test_wandb_log_random_samples_call_contract(tmp_path):
+    """NeMo-Skills' wandb usage (init/save/summary/finish) still matches the
+    wandb 0.28 API. Patched so it is deterministic and offline."""
+    from nemo_skills.inference import log_samples_wandb
+
+    jsonl = tmp_path / "samples.jsonl"
+    jsonl.write_text("\n".join(json.dumps({"problem": f"q{i}", "generation": f"a{i}"}) for i in range(4)) + "\n")
+
+    fake_wandb = MagicMock()
+    fake_wandb.summary = {}
+    with patch.object(log_samples_wandb, "wandb", fake_wandb):
+        log_samples_wandb.log_random_samples(str(jsonl), num_samples=2, project="regr", name="run1")
+
+    fake_wandb.init.assert_called_once()
+    assert fake_wandb.init.call_args.kwargs["project"] == "regr"
+    assert fake_wandb.init.call_args.kwargs["name"] == "run1"
+    fake_wandb.save.assert_called_once()
+    fake_wandb.finish.assert_called_once()
+    assert fake_wandb.summary["num_samples"] == 4
+
+
+def test_wandb_offline_init_runs(tmp_path, monkeypatch):
+    """Real wandb 0.28 imports and completes an offline init/finish cycle with no
+    account or network (guards the bundled wandb-core the CVE bump targets)."""
+    import wandb
+
+    monkeypatch.setenv("WANDB_MODE", "offline")
+    monkeypatch.setenv("WANDB_SILENT", "true")
+    monkeypatch.setenv("WANDB_DIR", str(tmp_path))
+    run = wandb.init(project="regr", name="offline-run", dir=str(tmp_path))
+    try:
+        run.log({"metric": 1.0})
+    finally:
+        wandb.finish()
+    assert (tmp_path / "wandb").exists()
+
+
+# ---------------------------------------------------------------------------
+# lxml >=6.1.0 — optional `stem` extra (skips when not installed, e.g. the
+# core-only unit-tests job); exercises the real parser when present.
+# ---------------------------------------------------------------------------
+
+
+def test_lxml_parses_html():
+    """lxml 6.1 parses HTML via both its own etree and as the BeautifulSoup
+    backend NeMo-Skills' `stem` extra provides. Skips cleanly when lxml is not
+    installed (it is not in the core/dev set)."""
+    lxml_html = pytest.importorskip("lxml.html")
+    tree = lxml_html.fromstring("<html><body><p id='x'>hello</p></body></html>")
+    assert tree.find(".//p").text == "hello"
+
+    bs4 = pytest.importorskip("bs4")
+    soup = bs4.BeautifulSoup("<html><body><p>hi</p></body></html>", "lxml")
+    assert soup.find("p").text == "hi"

From 2143dfc8a7bdbd3c6cc5de50702bbe39759bcd24 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 30 Jul 2026 11:46:44 -0400
Subject: [PATCH 05/30] fix(security): remediate nemo-skills high findings

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/requirements.txt               | 10 +++-
 dockerfiles/Dockerfile.nemo-skills  | 31 +++++++++-
 nemo_skills/inference/eval/bfcl.py  |  3 +-
 tests/test_dependency_functional.py | 30 +++++++++-
 tests/test_requirements_versions.py | 90 +++++++++++++++++++----------
 5 files changed, 127 insertions(+), 37 deletions(-)

diff --git a/core/requirements.txt b/core/requirements.txt
index ff5ff67111..09359976fa 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -7,12 +7,16 @@ bs4
 compute-eval @ git+https://github.com/NVIDIA/compute-eval.git@e01a5d2
 contractions
 datasets
+# Fixes eight High findings through CVE-2026-55415.
+datamodel-code-generator>=0.64.0
 editdistance
 evalplus @ git+https://github.com/evalplus/evalplus@c91370f
 faiss-cpu
 fire
 flask
 func-timeout
+# Fixes six High GitPython advisories through GHSA-94p4-4cq8-9g67.
+GitPython>=3.1.55
 gradio
 httpx
 huggingface_hub
@@ -47,6 +51,6 @@ sympy
 torchcodec
 tqdm
 transformers
-# Floors click to >=8.2 (wandb 0.26.x only requires click>=8.0.1, so the resolver
-# was settling on click 8.1.8). Taken from #1507.
-wandb>=0.27.1
+# 0.28.1 is paired with the patched wandb-core built in Dockerfile.nemo-skills.
+# Pinning keeps the Python/core protocol pair deterministic.
+wandb==0.28.1
diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 0819e58d51..1e9d5522da 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -1,3 +1,28 @@
+# W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, and x/text 0.38.0.
+# W&B commit e118409 is its focused July 17 dependency update to Go 1.26.5,
+# grpc-go 1.82.1, and x/text 0.40.0. Build only wandb-core here so the final
+# image gets the fixed binary without retaining the Go toolchain or source.
+ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
+FROM golang:1.26.5 AS wandb-core-builder
+ARG WANDB_CORE_COMMIT
+RUN git init /src/wandb && \
+    cd /src/wandb && \
+    git remote add origin https://github.com/wandb/wandb.git && \
+    git sparse-checkout init --cone && \
+    git sparse-checkout set core && \
+    git fetch --depth 1 origin "${WANDB_CORE_COMMIT}" && \
+    git checkout --detach FETCH_HEAD
+RUN cd /src/wandb/core && \
+    CGO_ENABLED=0 go build \
+        -tags "disable_grpc_modules parquet_read_only" \
+        -ldflags "-s -w -X main.commit=${WANDB_CORE_COMMIT}" \
+        -mod=vendor \
+        -o /wandb-core \
+        ./cmd/wandb-core && \
+    go version -m /wandb-core | grep -F "go1.26.5" && \
+    go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
+
 # using ubuntu instead of debian for easier apptainer installation on arm64
 FROM ubuntu:22.04
 
@@ -73,9 +98,13 @@ COPY core/requirements.txt /opt/NeMo-Skills/core/requirements.txt
 RUN pip install git+https://github.com/NVIDIA/NeMo-speech-data-processor@29b9b1ec0ceaf3ffa441c1d01297371b3f8e11d2
 ARG CACHEBUST=4
 # Install via `uv pip` from the project directory so [tool.uv].override-dependencies
-# in pyproject.toml (which relaxes leptonai's httpx==0.27.2 pin so litellm 1.83.x
+# in pyproject.toml (which relaxes leptonai's httpx==0.27.2 pin so litellm 1.84.x
 # can be installed) is picked up. Plain pip ignores [tool.uv] and the resolver fails.
 RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \
     -r core/requirements.txt -r requirements/pipeline.txt
+# Replace W&B's vulnerable release binary with the source-compatible patched core
+# built and module-verified above. The copy preserves its executable mode.
+COPY --from=wandb-core-builder /wandb-core /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core
+RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --version
 # Fix http mismatch between lepton and dggs by manually downloading dggs here
 RUN pip install ddgs
diff --git a/nemo_skills/inference/eval/bfcl.py b/nemo_skills/inference/eval/bfcl.py
index 7452aa14af..cff8a9e5fb 100644
--- a/nemo_skills/inference/eval/bfcl.py
+++ b/nemo_skills/inference/eval/bfcl.py
@@ -66,7 +66,8 @@
     "cohere==5.18.0",
     "typer>=0.12.5",
     "tabulate>=0.9.0",
-    "datamodel-code-generator==0.25.7",
+    # 0.64.0 fixes eight High findings through CVE-2026-55415.
+    "datamodel-code-generator==0.64.0",
     "google-genai>=1.52.0",
     # "qwen-agent",  # disabling due to some issues (and shouldn't be needed)
     "mpmath==1.3.0",
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 00dbc36f95..645f01752f 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -25,7 +25,9 @@
   * litellm[caching] ==1.84.10  (fixes GHSA-4xpc-pv4p-pm3w — the pre-1.84 client
                                  could leak the configured api_key to an
                                  attacker-controlled Host header)
-  * wandb            >=0.27.1
+  * GitPython        >=3.1.55
+  * datamodel-code-generator >=0.64.0
+  * wandb            ==0.28.1, with a patched core in the container
   * typer            >=0.16 / click cap removed  (typer<0.16 broke on click 8.2's
                                  Parameter.make_metavar signature — dynamo#1039)
   * lxml             >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw; optional `stem` extra,
@@ -37,11 +39,35 @@
 
 import asyncio
 import json
+from importlib.metadata import version
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
+# ---------------------------------------------------------------------------
+# GitPython >=3.1.55 and datamodel-code-generator >=0.64.0
+# ---------------------------------------------------------------------------
+
+
+def test_gitpython_can_initialize_and_inspect_repository(tmp_path):
+    import git
+    from packaging.version import Version
+
+    assert Version(version("GitPython")) >= Version("3.1.55")
+    repo = git.Repo.init(tmp_path)
+    assert not repo.bare
+    assert repo.git_dir == str(tmp_path / ".git")
+
+
+def test_datamodel_code_generator_imports_at_fixed_version():
+    import datamodel_code_generator
+    from packaging.version import Version
+
+    assert datamodel_code_generator is not None
+    assert Version(version("datamodel-code-generator")) >= Version("0.64.0")
+
+
 # ---------------------------------------------------------------------------
 # litellm 1.84.10 — driven through nemo_skills.inference.model
 # ---------------------------------------------------------------------------
@@ -203,7 +229,7 @@ def test_cli_unknown_command_is_usage_error(cli_app):
 
 
 # ---------------------------------------------------------------------------
-# wandb >=0.27.1 — driven through nemo_skills.inference.log_samples_wandb
+# wandb ==0.28.1 — driven through nemo_skills.inference.log_samples_wandb
 # ---------------------------------------------------------------------------
 
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 33becac1eb..b5dd25ef5d 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -16,7 +16,9 @@
 
 This PR bumps several dependency floors/pins to close known CVEs:
   * litellm[caching] -> ==1.84.10 (fixes GHSA-4xpc-pv4p-pm3w)
-  * wandb            -> >=0.27.1  (bundled wandb-core Go binary CVEs)
+  * GitPython        -> >=3.1.55  (fixes six High findings)
+  * datamodel-code-generator -> >=0.64.0 (fixes eight High findings)
+  * wandb            -> ==0.28.1, paired with a patched wandb-core
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
   * typer            -> >=0.16   (click 8.2 compatible)
   * click            -> pin removed from requirements/pipeline.txt
@@ -44,6 +46,8 @@
 PIPELINE_REQUIREMENTS = REPO_ROOT / "requirements" / "pipeline.txt"
 STEM_REQUIREMENTS = REPO_ROOT / "requirements" / "stem.txt"
 PYPROJECT_TOML = REPO_ROOT / "pyproject.toml"
+BFCL_MODULE = REPO_ROOT / "nemo_skills" / "inference" / "eval" / "bfcl.py"
+NEMO_SKILLS_DOCKERFILE = REPO_ROOT / "dockerfiles" / "Dockerfile.nemo-skills"
 
 
 def _load_toml(path: Path) -> dict:
@@ -85,10 +89,10 @@ def _find_requirement(path: Path, package_name: str) -> tuple[Requirement, str]:
 
 
 class TestCoreRequirements:
-    """core/requirements.txt: litellm and wandb security floors."""
+    """core/requirements.txt security floors and pins."""
 
     def test_litellm_pin_fixes_ghsa_4xpc_pv4p_pm3w(self):
-        req, comment = _find_requirement(CORE_REQUIREMENTS, "litellm")
+        req, _ = _find_requirement(CORE_REQUIREMENTS, "litellm")
         assert "caching" in req.extras, "litellm[caching] extra must be preserved"
 
         # Must be pinned to an exact version (== specifier) so the resolver is deterministic.
@@ -100,32 +104,68 @@ def test_litellm_pin_fixes_ghsa_4xpc_pv4p_pm3w(self):
             f"litellm is pinned to {pinned_version}, which is below the 1.84.0 floor that "
             "fixes GHSA-4xpc-pv4p-pm3w (API-key leak to arbitrary Host header)"
         )
-        assert "GHSA-4xpc-pv4p-pm3w" in comment
+        assert "GHSA-4xpc-pv4p-pm3w" in CORE_REQUIREMENTS.read_text()
 
     def test_litellm_vulnerable_pin_not_reintroduced(self):
         """Regression guard: the old vulnerable exact pin must not come back."""
         content = CORE_REQUIREMENTS.read_text()
         assert "litellm[caching]==1.83.14" not in content
-        assert "1.83.14" not in content
 
-    def test_wandb_pin_fixes_bundled_go_binary_cves(self):
-        req, comment = _find_requirement(CORE_REQUIREMENTS, "wandb")
+    def test_gitpython_floor_fixes_all_six_high_findings(self):
+        req, _ = _find_requirement(CORE_REQUIREMENTS, "GitPython")
         specs = {spec.operator: spec.version for spec in req.specifier}
-        assert ">=" in specs, f"expected a floor (>=) specifier for wandb, got {req.specifier}"
+        assert ">=" in specs, f"expected a floor (>=) specifier for GitPython, got {req.specifier}"
+        assert Version(specs[">="]) >= Version("3.1.55")
 
-        floor_version = Version(specs[">="])
-        assert floor_version >= Version("0.27.1"), (
-            f"wandb floor is {floor_version}, which is below 0.27.1 (first release with the "
-            "patched x/crypto 0.52.0 wandb-core binary)"
-        )
-        assert "click>=8.2" in comment
+    def test_datamodel_code_generator_floor_fixes_all_eight_high_findings(self):
+        req, _ = _find_requirement(CORE_REQUIREMENTS, "datamodel-code-generator")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for datamodel-code-generator, got {req.specifier}"
+        assert Version(specs[">="]) >= Version("0.64.0")
+
+    def test_bfcl_does_not_reinstall_vulnerable_datamodel_code_generator(self):
+        content = BFCL_MODULE.read_text()
+        assert '"datamodel-code-generator==0.64.0"' in content
+        assert "datamodel-code-generator==0.25.7" not in content
+
+    def test_wandb_python_version_is_pinned_to_patched_core_pair(self):
+        req, _ = _find_requirement(CORE_REQUIREMENTS, "wandb")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert specs == {"==": "0.28.1"}
 
     def test_wandb_is_not_unbounded_or_unpinned(self):
-        """wandb must remain a floor-pinned requirement (bare 'wandb' with no version is
-        the pre-fix state and would allow an unvetted, potentially vulnerable version)."""
+        """A bare wandb requirement could resolve to a release with a vulnerable core."""
         for raw_line, code_part, _ in _iter_requirement_lines(CORE_REQUIREMENTS):
             if code_part == "wandb":
-                pytest.fail(f"wandb requirement has no version floor: {raw_line!r}")
+                pytest.fail(f"wandb requirement has no version pin: {raw_line!r}")
+
+
+class TestPatchedWandbCoreDockerBuild:
+    """The final image must replace and verify W&B's bundled Go executable."""
+
+    @pytest.fixture(scope="class")
+    def dockerfile(self):
+        return NEMO_SKILLS_DOCKERFILE.read_text()
+
+    def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
+        assert "WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7" in dockerfile
+
+    @pytest.mark.parametrize(
+        "expected",
+        [
+            "FROM golang:1.26.5 AS wandb-core-builder",
+            'go version -m /wandb-core | grep -F "go1.26.5"',
+            "google\\.golang\\.org/grpc[[:space:]]+v1\\.82\\.1",
+            "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",
+        ],
+    )
+    def test_fixed_go_components_are_build_time_verified(self, dockerfile, expected):
+        assert expected in dockerfile
+
+    def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
+        destination = "/usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core"
+        assert f"COPY --from=wandb-core-builder /wandb-core {destination}" in dockerfile
+        assert f"RUN {destination} --version" in dockerfile
 
 
 class TestPipelineRequirements:
@@ -152,16 +192,16 @@ def test_click_pin_line_absent_from_raw_text(self):
         assert not re.search(r"^click\s*[<>=]", content, re.MULTILINE)
 
     def test_typer_floor_is_click_8_2_compatible(self):
-        req, comment = _find_requirement(PIPELINE_REQUIREMENTS, "typer")
+        req, _ = _find_requirement(PIPELINE_REQUIREMENTS, "typer")
         specs = {spec.operator: spec.version for spec in req.specifier}
         assert ">=" in specs, f"expected a floor (>=) specifier for typer, got {req.specifier}"
 
         floor_version = Version(specs[">="])
         assert floor_version >= Version("0.16"), (
             f"typer floor is {floor_version}, which is below 0.16 (the first click-8.2-compatible "
-            "release that also satisfies wandb>=0.27.1's click>=8.2 requirement)"
+            "release for click 8.2)"
         )
-        assert "click 8.2" in comment or "click>=8.2" in comment
+        assert "click 8.2" in PIPELINE_REQUIREMENTS.read_text()
 
     def test_nemo_run_and_launcher_pins_untouched(self):
         """Sanity: other pipeline deps referenced by the diff context are still present."""
@@ -247,13 +287,3 @@ def test_pipeline_requirements_lines_are_parseable():
             Requirement(code_part)
         except InvalidRequirement as exc:
             pytest.fail(f"Unparseable requirement line {raw_line!r}: {exc}")
-
-
-def test_wandb_and_typer_click_requirement_comments_are_consistent():
-    """wandb's comment says it needs click>=8.2; typer's comment (in the
-    sibling pipeline.txt file) should agree, since typer>=0.16 is the
-    mechanism that satisfies that click floor without conflicting pins."""
-    _, wandb_comment = _find_requirement(CORE_REQUIREMENTS, "wandb")
-    _, typer_comment = _find_requirement(PIPELINE_REQUIREMENTS, "typer")
-    assert "click>=8.2" in wandb_comment
-    assert "click 8.2" in typer_comment

From 5672da1fefb003402f6c1a04c219326933bab09b Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 30 Jul 2026 11:50:18 -0400
Subject: [PATCH 06/30] chore: satisfy requirements sorting hook

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/requirements.txt b/core/requirements.txt
index 09359976fa..1c707e6e86 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -6,9 +6,9 @@
 bs4
 compute-eval @ git+https://github.com/NVIDIA/compute-eval.git@e01a5d2
 contractions
-datasets
 # Fixes eight High findings through CVE-2026-55415.
 datamodel-code-generator>=0.64.0
+datasets
 editdistance
 evalplus @ git+https://github.com/evalplus/evalplus@c91370f
 faiss-cpu

From 90da975076527305c19a08dae93fd231a7bd3a49 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 30 Jul 2026 11:55:42 -0400
Subject: [PATCH 07/30] fix(container): validate patched wandb core help output

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 4 +++-
 tests/test_requirements_versions.py | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 1e9d5522da..b936cf27a7 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -25,6 +25,7 @@ RUN cd /src/wandb/core && \
 
 # using ubuntu instead of debian for easier apptainer installation on arm64
 FROM ubuntu:22.04
+ARG WANDB_CORE_COMMIT
 
 # Install Python and other dependencies
 RUN apt-get update && \
@@ -105,6 +106,7 @@ RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \
 # Replace W&B's vulnerable release binary with the source-compatible patched core
 # built and module-verified above. The copy preserves its executable mode.
 COPY --from=wandb-core-builder /wandb-core /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core
-RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --version
+RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --help 2>&1 | \
+    grep -F "Commit SHA: ${WANDB_CORE_COMMIT}"
 # Fix http mismatch between lepton and dggs by manually downloading dggs here
 RUN pip install ddgs
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index b5dd25ef5d..faf0bb9743 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -165,7 +165,8 @@ def test_fixed_go_components_are_build_time_verified(self, dockerfile, expected)
     def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
         destination = "/usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core"
         assert f"COPY --from=wandb-core-builder /wandb-core {destination}" in dockerfile
-        assert f"RUN {destination} --version" in dockerfile
+        assert f"RUN {destination} --help 2>&1" in dockerfile
+        assert 'grep -F "Commit SHA: ${WANDB_CORE_COMMIT}"' in dockerfile
 
 
 class TestPipelineRequirements:

From 888bd648824b60ec19c28f88b09db985e5c4509e Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 30 Jul 2026 16:24:27 -0400
Subject: [PATCH 08/30] fix(container): remove uv Git cache from runtime image

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 4 ++++
 tests/test_requirements_versions.py | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index b936cf27a7..1e6ef71c8c 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -110,3 +110,7 @@ RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --help 2>&1 | \
     grep -F "Commit SHA: ${WANDB_CORE_COMMIT}"
 # Fix http mismatch between lepton and dggs by manually downloading dggs here
 RUN pip install ddgs
+
+# nSpect's global policy flags Git metadata left by uv's source-distribution
+# cache. The cache is build-only, so remove it from the published image.
+RUN rm -rf /root/.cache/uv
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index faf0bb9743..df76eaf79c 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -168,6 +168,9 @@ def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
         assert f"RUN {destination} --help 2>&1" in dockerfile
         assert 'grep -F "Commit SHA: ${WANDB_CORE_COMMIT}"' in dockerfile
 
+    def test_uv_git_cache_is_removed_from_final_image(self, dockerfile):
+        assert "RUN rm -rf /root/.cache/uv" in dockerfile
+
 
 class TestPipelineRequirements:
     """requirements/pipeline.txt: click unpin + typer floor."""

From 579c6d6a5558de23bb201ba1ed8f601dd73cc8f7 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 31 Jul 2026 10:26:13 -0400
Subject: [PATCH 09/30] fix(security): floor msgpack and setuptools

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/pyproject.toml                 |  2 +-
 dockerfiles/Dockerfile.nemo-skills  |  9 ++++++++-
 pyproject.toml                      |  7 ++++++-
 tests/test_dependency_functional.py |  9 +++++++++
 tests/test_requirements_versions.py | 28 ++++++++++++++++++++++++++++
 tools/pyproject.toml                |  2 +-
 6 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/core/pyproject.toml b/core/pyproject.toml
index c5ccb3bd1f..0cc768c30d 100644
--- a/core/pyproject.toml
+++ b/core/pyproject.toml
@@ -14,7 +14,7 @@
 
 [build-system]
 requires = [
-    "setuptools",
+    "setuptools>=78.1.1",
     "wheel"
 ]
 build-backend = "setuptools.build_meta"
diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 1e6ef71c8c..2942ea852c 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -40,7 +40,7 @@ RUN apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip setuptools "uv>=0.11.10"
+RUN pip install --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10"
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
@@ -111,6 +111,13 @@ RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --help 2>&1 | \
 # Fix http mismatch between lepton and dggs by manually downloading dggs here
 RUN pip install ddgs
 
+# Guard the final resolved environment against the two High findings seen in
+# the July 31 multi-architecture image scan.
+RUN python -c "from importlib.metadata import version as v; from packaging.version import Version as V; \
+               assert V(v('msgpack')) >= V('1.2.1'), v('msgpack'); \
+               assert V(v('setuptools')) >= V('78.1.1'), v('setuptools'); \
+               print('msgpack/setuptools security floors OK')"
+
 # nSpect's global policy flags Git metadata left by uv's source-distribution
 # cache. The cache is build-only, so remove it from the published image.
 RUN rm -rf /root/.cache/uv
diff --git a/pyproject.toml b/pyproject.toml
index 617e40e6e9..a3423865ae 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,7 +14,7 @@
 
 [build-system]
 requires = [
-    "setuptools",
+    "setuptools>=78.1.1",
     "wheel"
 ]
 build-backend = "setuptools.build_meta"
@@ -80,6 +80,11 @@ override-dependencies = [
     # 0.7.0 (a transitive dep of nemo_run) pins urllib3<1.27, but in practice
     # urllib3>=2 works at runtime, so override the constraint.
     "urllib3>=2.6.3",
+    # Container scans found msgpack 1.1.2 (GHSA-6v7p-g79w-8964) and setuptools
+    # 70.3.0 (CVE-2025-47273) after transitive resolution. Keep the complete
+    # environment above the first fixed releases.
+    "msgpack>=1.2.1",
+    "setuptools>=78.1.1",
 ]
 
 [tool.pytest.ini_options]
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 645f01752f..299db4e97c 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -32,6 +32,8 @@
                                  Parameter.make_metavar signature — dynamo#1039)
   * lxml             >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw; optional `stem` extra,
                                so guarded by importorskip)
+  * msgpack          >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
+  * setuptools       >=78.1.1 (fixes CVE-2025-47273)
 
 All tests are CPU-only, hermetic (no sandbox container, no live LLM endpoint, no
 API keys) so they run in the existing `unit-tests` (`-m "not gpu"`) CI job.
@@ -68,6 +70,13 @@ def test_datamodel_code_generator_imports_at_fixed_version():
     assert Version(version("datamodel-code-generator")) >= Version("0.64.0")
 
 
+def test_msgpack_and_setuptools_security_floors():
+    from packaging.version import Version
+
+    assert Version(version("msgpack")) >= Version("1.2.1")
+    assert Version(version("setuptools")) >= Version("78.1.1")
+
+
 # ---------------------------------------------------------------------------
 # litellm 1.84.10 — driven through nemo_skills.inference.model
 # ---------------------------------------------------------------------------
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index df76eaf79c..64197c8385 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -20,6 +20,8 @@
   * datamodel-code-generator -> >=0.64.0 (fixes eight High findings)
   * wandb            -> ==0.28.1, paired with a patched wandb-core
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
+  * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
+  * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
   * typer            -> >=0.16   (click 8.2 compatible)
   * click            -> pin removed from requirements/pipeline.txt
 
@@ -48,6 +50,7 @@
 PYPROJECT_TOML = REPO_ROOT / "pyproject.toml"
 BFCL_MODULE = REPO_ROOT / "nemo_skills" / "inference" / "eval" / "bfcl.py"
 NEMO_SKILLS_DOCKERFILE = REPO_ROOT / "dockerfiles" / "Dockerfile.nemo-skills"
+BUILD_PYPROJECTS = [PYPROJECT_TOML, REPO_ROOT / "core" / "pyproject.toml", REPO_ROOT / "tools" / "pyproject.toml"]
 
 
 def _load_toml(path: Path) -> dict:
@@ -171,6 +174,10 @@ def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
     def test_uv_git_cache_is_removed_from_final_image(self, dockerfile):
         assert "RUN rm -rf /root/.cache/uv" in dockerfile
 
+    def test_final_image_asserts_msgpack_and_setuptools_floors(self, dockerfile):
+        assert "V(v('msgpack')) >= V('1.2.1')" in dockerfile
+        assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
+
 
 class TestPipelineRequirements:
     """requirements/pipeline.txt: click unpin + typer floor."""
@@ -263,6 +270,16 @@ def test_urllib3_override_present(self, uv_overrides):
         assert ">=" in specs
         assert Version(specs[">="]) >= Version("2.6.3")
 
+    @pytest.mark.parametrize(
+        ("package", "minimum"),
+        [("msgpack", "1.2.1"), ("setuptools", "78.1.1")],
+    )
+    def test_container_security_override_present(self, uv_overrides, package, minimum):
+        assert package in uv_overrides
+        specs = {spec.operator: spec.version for spec in uv_overrides[package].specifier}
+        assert ">=" in specs
+        assert Version(specs[">="]) >= Version(minimum)
+
     def test_dependencies_still_sourced_from_core_and_pipeline_requirements(self):
         data = _load_toml(PYPROJECT_TOML)
         dynamic_deps = data["tool"]["setuptools"]["dynamic"]["dependencies"]
@@ -291,3 +308,14 @@ def test_pipeline_requirements_lines_are_parseable():
             Requirement(code_part)
         except InvalidRequirement as exc:
             pytest.fail(f"Unparseable requirement line {raw_line!r}: {exc}")
+
+
+@pytest.mark.parametrize("pyproject", BUILD_PYPROJECTS)
+def test_setuptools_build_floor_fixes_cve_2025_47273(pyproject):
+    data = _load_toml(pyproject)
+    requirement = next(
+        Requirement(entry) for entry in data["build-system"]["requires"] if Requirement(entry).name == "setuptools"
+    )
+    specs = {spec.operator: spec.version for spec in requirement.specifier}
+    assert ">=" in specs
+    assert Version(specs[">="]) >= Version("78.1.1")
diff --git a/tools/pyproject.toml b/tools/pyproject.toml
index 4e9c73ee71..1cda69d7ca 100644
--- a/tools/pyproject.toml
+++ b/tools/pyproject.toml
@@ -14,7 +14,7 @@
 
 [build-system]
 requires = [
-    "setuptools",
+    "setuptools>=78.1.1",
     "wheel"
 ]
 build-backend = "setuptools.build_meta"

From b90f97701d9b8ec4c5de9ede0b20d3df5e03cdb4 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 6 Aug 2026 12:11:57 -0400
Subject: [PATCH 10/30] fix(container): remediate aiohttp Trivy high

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 21 ++++++++++++++++++---
 pyproject.toml                      |  4 ++--
 tests/test_dependency_functional.py |  4 +++-
 tests/test_requirements_versions.py | 10 ++++++++--
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 2942ea852c..07371c66d2 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -111,12 +111,27 @@ RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --help 2>&1 | \
 # Fix http mismatch between lepton and dggs by manually downloading dggs here
 RUN pip install ddgs
 
-# Guard the final resolved environment against the two High findings seen in
-# the July 31 multi-architecture image scan.
+# Ray's runtime-env agent prepends its bundled aiohttp 3.14.1 over the fixed
+# environment package. Replace that private copy with the resolved 3.14.3 files
+# from the same architecture (CVE-2026-69244). uv also packages a build SBOM
+# containing msgpack 1.1.2 and setuptools 70.3.0; remove that non-runtime SBOM so
+# scanners do not report uv's build dependencies as installed packages.
+RUN site_packages="$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")" && \
+    ray_thirdparty="${site_packages}/ray/_private/runtime_env/agent/thirdparty_files" && \
+    rm -rf "${ray_thirdparty}/aiohttp" "${ray_thirdparty}"/aiohttp-*.dist-info && \
+    cp -a "${site_packages}/aiohttp" "${ray_thirdparty}/" && \
+    cp -a "${site_packages}"/aiohttp-*.dist-info "${ray_thirdparty}/" && \
+    find "${site_packages}" -type d -path '*/uv-*.dist-info/sboms' -prune -exec rm -rf {} +
+
+# Guard the final resolved environment and Ray's private import path against the
+# High findings seen in the multi-architecture image scans.
 RUN python -c "from importlib.metadata import version as v; from packaging.version import Version as V; \
+               assert V(v('aiohttp')) >= V('3.14.3'), v('aiohttp'); \
                assert V(v('msgpack')) >= V('1.2.1'), v('msgpack'); \
                assert V(v('setuptools')) >= V('78.1.1'), v('setuptools'); \
-               print('msgpack/setuptools security floors OK')"
+               print('aiohttp/msgpack/setuptools security floors OK')" && \
+    python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
+    ! find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' | grep -q .
 
 # nSpect's global policy flags Git metadata left by uv's source-distribution
 # cache. The cache is build-only, so remove it from the published image.
diff --git a/pyproject.toml b/pyproject.toml
index a3423865ae..7076b965b4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,9 +80,9 @@ override-dependencies = [
     # 0.7.0 (a transitive dep of nemo_run) pins urllib3<1.27, but in practice
     # urllib3>=2 works at runtime, so override the constraint.
     "urllib3>=2.6.3",
-    # Container scans found msgpack 1.1.2 (GHSA-6v7p-g79w-8964) and setuptools
-    # 70.3.0 (CVE-2025-47273) after transitive resolution. Keep the complete
+    # Container scans found vulnerable transitive versions. Keep the complete
     # environment above the first fixed releases.
+    "aiohttp>=3.14.3",
     "msgpack>=1.2.1",
     "setuptools>=78.1.1",
 ]
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 299db4e97c..74cbda75a0 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -32,6 +32,7 @@
                                  Parameter.make_metavar signature — dynamo#1039)
   * lxml             >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw; optional `stem` extra,
                                so guarded by importorskip)
+  * aiohttp          >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
   * setuptools       >=78.1.1 (fixes CVE-2025-47273)
 
@@ -70,9 +71,10 @@ def test_datamodel_code_generator_imports_at_fixed_version():
     assert Version(version("datamodel-code-generator")) >= Version("0.64.0")
 
 
-def test_msgpack_and_setuptools_security_floors():
+def test_aiohttp_msgpack_and_setuptools_security_floors():
     from packaging.version import Version
 
+    assert Version(version("aiohttp")) >= Version("3.14.3")
     assert Version(version("msgpack")) >= Version("1.2.1")
     assert Version(version("setuptools")) >= Version("78.1.1")
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 64197c8385..4af679516f 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -20,6 +20,7 @@
   * datamodel-code-generator -> >=0.64.0 (fixes eight High findings)
   * wandb            -> ==0.28.1, paired with a patched wandb-core
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
+  * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
   * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
   * typer            -> >=0.16   (click 8.2 compatible)
@@ -174,10 +175,15 @@ def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
     def test_uv_git_cache_is_removed_from_final_image(self, dockerfile):
         assert "RUN rm -rf /root/.cache/uv" in dockerfile
 
-    def test_final_image_asserts_msgpack_and_setuptools_floors(self, dockerfile):
+    def test_final_image_asserts_python_security_floors(self, dockerfile):
+        assert "V(v('aiohttp')) >= V('3.14.3')" in dockerfile
         assert "V(v('msgpack')) >= V('1.2.1')" in dockerfile
         assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
 
+    def test_ray_private_aiohttp_and_uv_build_sbom_are_remediated(self, dockerfile):
+        assert "ray/_private/runtime_env/agent/thirdparty_files" in dockerfile
+        assert "uv-*.dist-info/sboms" in dockerfile
+
 
 class TestPipelineRequirements:
     """requirements/pipeline.txt: click unpin + typer floor."""
@@ -272,7 +278,7 @@ def test_urllib3_override_present(self, uv_overrides):
 
     @pytest.mark.parametrize(
         ("package", "minimum"),
-        [("msgpack", "1.2.1"), ("setuptools", "78.1.1")],
+        [("aiohttp", "3.14.3"), ("msgpack", "1.2.1"), ("setuptools", "78.1.1")],
     )
     def test_container_security_override_present(self, uv_overrides, package, minimum):
         assert package in uv_overrides

From 91ef6b6f83e611b380228f2a644dd66f5bdd968d Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Mon, 10 Aug 2026 12:02:07 -0400
Subject: [PATCH 11/30] fix(container): update wandb core for go-git CVE

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 11 ++++++-----
 tests/test_requirements_versions.py |  4 +++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 07371c66d2..b795c43040 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -1,8 +1,8 @@
-# W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, and x/text 0.38.0.
-# W&B commit e118409 is its focused July 17 dependency update to Go 1.26.5,
-# grpc-go 1.82.1, and x/text 0.40.0. Build only wandb-core here so the final
-# image gets the fixed binary without retaining the Go toolchain or source.
-ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
+# W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, x/text 0.38.0, and
+# go-git 5.19.1. W&B commit 16af7d3 includes the prior dependency updates plus
+# go-git 5.19.2, which fixes CVE-2026-71556. Build only wandb-core here so the
+# final image gets the fixed binary without retaining the Go toolchain or source.
+ARG WANDB_CORE_COMMIT=16af7d3b52deacaa7c1ed521ffb5c5941d9df9f4
 FROM golang:1.26.5 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
 RUN git init /src/wandb && \
@@ -20,6 +20,7 @@ RUN cd /src/wandb/core && \
         -o /wandb-core \
         ./cmd/wandb-core && \
     go version -m /wandb-core | grep -F "go1.26.5" && \
+    go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v5\.19\.2([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 4af679516f..30ebe10236 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -19,6 +19,7 @@
   * GitPython        -> >=3.1.55  (fixes six High findings)
   * datamodel-code-generator -> >=0.64.0 (fixes eight High findings)
   * wandb            -> ==0.28.1, paired with a patched wandb-core
+                         (fixes CVE-2026-71556 in bundled go-git)
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
@@ -152,13 +153,14 @@ def dockerfile(self):
         return NEMO_SKILLS_DOCKERFILE.read_text()
 
     def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
-        assert "WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7" in dockerfile
+        assert "WANDB_CORE_COMMIT=16af7d3b52deacaa7c1ed521ffb5c5941d9df9f4" in dockerfile
 
     @pytest.mark.parametrize(
         "expected",
         [
             "FROM golang:1.26.5 AS wandb-core-builder",
             'go version -m /wandb-core | grep -F "go1.26.5"',
+            "github\\.com/go-git/go-git/v5[[:space:]]+v5\\.19\\.2",
             "google\\.golang\\.org/grpc[[:space:]]+v1\\.82\\.1",
             "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",
         ],

From 141783767bfea696d9407355c7099dfdafb60bf3 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Mon, 10 Aug 2026 12:05:55 -0400
Subject: [PATCH 12/30] fix(container): narrowly backport go-git update

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 16 +++++++++++-----
 tests/test_requirements_versions.py |  9 +++++++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index b795c43040..39e0b7bdb3 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -1,10 +1,12 @@
 # W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, x/text 0.38.0, and
-# go-git 5.19.1. W&B commit 16af7d3 includes the prior dependency updates plus
-# go-git 5.19.2, which fixes CVE-2026-71556. Build only wandb-core here so the
-# final image gets the fixed binary without retaining the Go toolchain or source.
-ARG WANDB_CORE_COMMIT=16af7d3b52deacaa7c1ed521ffb5c5941d9df9f4
+# go-git 5.19.1. Keep the qualified W&B dependency-update commit and minimally
+# backport W&B PR #12326's go-git 5.19.2 fix for CVE-2026-71556. Build only
+# wandb-core so the final image does not retain the Go toolchain or source.
+ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
+ARG WANDB_GO_GIT_VERSION=5.19.2
 FROM golang:1.26.5 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
+ARG WANDB_GO_GIT_VERSION
 RUN git init /src/wandb && \
     cd /src/wandb && \
     git remote add origin https://github.com/wandb/wandb.git && \
@@ -13,6 +15,10 @@ RUN git init /src/wandb && \
     git fetch --depth 1 origin "${WANDB_CORE_COMMIT}" && \
     git checkout --detach FETCH_HEAD
 RUN cd /src/wandb/core && \
+    go get "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}" && \
+    go mod vendor && \
+    grep -E "^[[:space:]]*github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" go.mod && \
+    grep -E "^# github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" vendor/modules.txt && \
     CGO_ENABLED=0 go build \
         -tags "disable_grpc_modules parquet_read_only" \
         -ldflags "-s -w -X main.commit=${WANDB_CORE_COMMIT}" \
@@ -20,7 +26,7 @@ RUN cd /src/wandb/core && \
         -o /wandb-core \
         ./cmd/wandb-core && \
     go version -m /wandb-core | grep -F "go1.26.5" && \
-    go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v5\.19\.2([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 30ebe10236..1da8ef0c30 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -153,14 +153,19 @@ def dockerfile(self):
         return NEMO_SKILLS_DOCKERFILE.read_text()
 
     def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
-        assert "WANDB_CORE_COMMIT=16af7d3b52deacaa7c1ed521ffb5c5941d9df9f4" in dockerfile
+        assert "WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7" in dockerfile
+        assert "WANDB_GO_GIT_VERSION=5.19.2" in dockerfile
 
     @pytest.mark.parametrize(
         "expected",
         [
             "FROM golang:1.26.5 AS wandb-core-builder",
+            'go get "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}"',
+            "go mod vendor",
+            "go.mod",
+            "vendor/modules.txt",
             'go version -m /wandb-core | grep -F "go1.26.5"',
-            "github\\.com/go-git/go-git/v5[[:space:]]+v5\\.19\\.2",
+            "github\\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}",
             "google\\.golang\\.org/grpc[[:space:]]+v1\\.82\\.1",
             "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",
         ],

From d8118a13874f9d3c62b77befff81f8090db30c16 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Wed, 19 Aug 2026 16:20:29 -0400
Subject: [PATCH 13/30] fix: update Go and GitPython security floors

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/requirements.txt               |  4 ++--
 dockerfiles/Dockerfile.nemo-skills  |  9 +++++----
 tests/test_dependency_functional.py |  6 +++---
 tests/test_requirements_versions.py | 12 ++++++------
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/core/requirements.txt b/core/requirements.txt
index 1c707e6e86..1d3f3ac7c2 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -15,8 +15,8 @@ faiss-cpu
 fire
 flask
 func-timeout
-# Fixes six High GitPython advisories through GHSA-94p4-4cq8-9g67.
-GitPython>=3.1.55
+# Fixes all High GitPython advisories reported through GHSA-wvpp-8hx9-p66j.
+GitPython>=3.1.58
 gradio
 httpx
 huggingface_hub
diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 39e0b7bdb3..d766949a18 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -1,10 +1,11 @@
 # W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, x/text 0.38.0, and
 # go-git 5.19.1. Keep the qualified W&B dependency-update commit and minimally
-# backport W&B PR #12326's go-git 5.19.2 fix for CVE-2026-71556. Build only
-# wandb-core so the final image does not retain the Go toolchain or source.
+# backport W&B PR #12326's go-git 5.19.2 fix for CVE-2026-71556. Go 1.26.6
+# also clears the reported Go standard-library findings. Build only wandb-core
+# so the final image does not retain the Go toolchain or source.
 ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
 ARG WANDB_GO_GIT_VERSION=5.19.2
-FROM golang:1.26.5 AS wandb-core-builder
+FROM golang:1.26.6 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
 ARG WANDB_GO_GIT_VERSION
 RUN git init /src/wandb && \
@@ -25,7 +26,7 @@ RUN cd /src/wandb/core && \
         -mod=vendor \
         -o /wandb-core \
         ./cmd/wandb-core && \
-    go version -m /wandb-core | grep -F "go1.26.5" && \
+    go version -m /wandb-core | grep -F "go1.26.6" && \
     go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 74cbda75a0..f0d53f3bd0 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -25,7 +25,7 @@
   * litellm[caching] ==1.84.10  (fixes GHSA-4xpc-pv4p-pm3w — the pre-1.84 client
                                  could leak the configured api_key to an
                                  attacker-controlled Host header)
-  * GitPython        >=3.1.55
+  * GitPython        >=3.1.58
   * datamodel-code-generator >=0.64.0
   * wandb            ==0.28.1, with a patched core in the container
   * typer            >=0.16 / click cap removed  (typer<0.16 broke on click 8.2's
@@ -49,7 +49,7 @@
 import pytest
 
 # ---------------------------------------------------------------------------
-# GitPython >=3.1.55 and datamodel-code-generator >=0.64.0
+# GitPython >=3.1.58 and datamodel-code-generator >=0.64.0
 # ---------------------------------------------------------------------------
 
 
@@ -57,7 +57,7 @@ def test_gitpython_can_initialize_and_inspect_repository(tmp_path):
     import git
     from packaging.version import Version
 
-    assert Version(version("GitPython")) >= Version("3.1.55")
+    assert Version(version("GitPython")) >= Version("3.1.58")
     repo = git.Repo.init(tmp_path)
     assert not repo.bare
     assert repo.git_dir == str(tmp_path / ".git")
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 1da8ef0c30..0bce7d3ede 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -16,10 +16,10 @@
 
 This PR bumps several dependency floors/pins to close known CVEs:
   * litellm[caching] -> ==1.84.10 (fixes GHSA-4xpc-pv4p-pm3w)
-  * GitPython        -> >=3.1.55  (fixes six High findings)
+  * GitPython        -> >=3.1.58  (fixes all currently reported High findings)
   * datamodel-code-generator -> >=0.64.0 (fixes eight High findings)
   * wandb            -> ==0.28.1, paired with a patched wandb-core
-                         (fixes CVE-2026-71556 in bundled go-git)
+                         (fixes bundled go-git and Go stdlib findings)
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
@@ -116,11 +116,11 @@ def test_litellm_vulnerable_pin_not_reintroduced(self):
         content = CORE_REQUIREMENTS.read_text()
         assert "litellm[caching]==1.83.14" not in content
 
-    def test_gitpython_floor_fixes_all_six_high_findings(self):
+    def test_gitpython_floor_fixes_all_reported_high_findings(self):
         req, _ = _find_requirement(CORE_REQUIREMENTS, "GitPython")
         specs = {spec.operator: spec.version for spec in req.specifier}
         assert ">=" in specs, f"expected a floor (>=) specifier for GitPython, got {req.specifier}"
-        assert Version(specs[">="]) >= Version("3.1.55")
+        assert Version(specs[">="]) >= Version("3.1.58")
 
     def test_datamodel_code_generator_floor_fixes_all_eight_high_findings(self):
         req, _ = _find_requirement(CORE_REQUIREMENTS, "datamodel-code-generator")
@@ -159,12 +159,12 @@ def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
     @pytest.mark.parametrize(
         "expected",
         [
-            "FROM golang:1.26.5 AS wandb-core-builder",
+            "FROM golang:1.26.6 AS wandb-core-builder",
             'go get "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}"',
             "go mod vendor",
             "go.mod",
             "vendor/modules.txt",
-            'go version -m /wandb-core | grep -F "go1.26.5"',
+            'go version -m /wandb-core | grep -F "go1.26.6"',
             "github\\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}",
             "google\\.golang\\.org/grpc[[:space:]]+v1\\.82\\.1",
             "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",

From 03a4dba9ad67b5ba2c6f8f4849fced97034e9931 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 13:44:31 -0400
Subject: [PATCH 14/30] fix(security): refresh container dependency floors

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 29 +++++++++++++++++++++--------
 pyproject.toml                      |  1 +
 requirements/stem.txt               |  2 +-
 tests/test_dependency_functional.py |  4 +++-
 tests/test_requirements_versions.py | 22 ++++++++++++++++++----
 5 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index d766949a18..d9baf78488 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -1,13 +1,19 @@
 # W&B 0.28.1 still bundles Go 1.26.4, grpc-go 1.82.0, x/text 0.38.0, and
-# go-git 5.19.1. Keep the qualified W&B dependency-update commit and minimally
-# backport W&B PR #12326's go-git 5.19.2 fix for CVE-2026-71556. Go 1.26.6
-# also clears the reported Go standard-library findings. Build only wandb-core
-# so the final image does not retain the Go toolchain or source.
+# go-git 5.19.1. Keep the qualified W&B dependency-update commit and raise only
+# the modules with fixable Critical/High findings. Go 1.26.6 also clears the
+# reported Go standard-library findings. Build only wandb-core so the final
+# image does not retain the Go toolchain or source.
 ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
 ARG WANDB_GO_GIT_VERSION=5.19.2
+ARG WANDB_GO_CRYPTO_VERSION=0.55.0
+ARG WANDB_GO_IMAGE_VERSION=0.45.0
+ARG WANDB_GRPC_VERSION=1.83.1
 FROM golang:1.26.6 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
 ARG WANDB_GO_GIT_VERSION
+ARG WANDB_GO_CRYPTO_VERSION
+ARG WANDB_GO_IMAGE_VERSION
+ARG WANDB_GRPC_VERSION
 RUN git init /src/wandb && \
     cd /src/wandb && \
     git remote add origin https://github.com/wandb/wandb.git && \
@@ -16,7 +22,11 @@ RUN git init /src/wandb && \
     git fetch --depth 1 origin "${WANDB_CORE_COMMIT}" && \
     git checkout --detach FETCH_HEAD
 RUN cd /src/wandb/core && \
-    go get "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}" && \
+    go get \
+        "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}" \
+        "golang.org/x/crypto@v${WANDB_GO_CRYPTO_VERSION}" \
+        "golang.org/x/image@v${WANDB_GO_IMAGE_VERSION}" \
+        "google.golang.org/grpc@v${WANDB_GRPC_VERSION}" && \
     go mod vendor && \
     grep -E "^[[:space:]]*github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" go.mod && \
     grep -E "^# github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" vendor/modules.txt && \
@@ -28,7 +38,9 @@ RUN cd /src/wandb/core && \
         ./cmd/wandb-core && \
     go version -m /wandb-core | grep -F "go1.26.6" && \
     go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" && \
-    go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "golang\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "golang\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v${WANDB_GRPC_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
 
 # using ubuntu instead of debian for easier apptainer installation on arm64
@@ -93,7 +105,7 @@ RUN cd ${IFBENCH_DIR} && pip install -r requirements.txt
 COPY dockerfiles/ifbench.patch /opt/benchmarks/IFBench/ifbench.patch
 RUN cd /opt/benchmarks/IFBench && git apply ifbench.patch
 
-RUN pip install langdetect absl-py immutabledict nltk ipython && \
+RUN pip install langdetect absl-py immutabledict "nltk>=3.10.3" ipython && \
     python -c "import nltk; from spacy.cli import download; nltk.download('punkt'); nltk.download('punkt_tab'); \
     nltk.download('stopwords'); nltk.download('averaged_perceptron_tagger_eng'); download('en_core_web_sm')"
 
@@ -136,8 +148,9 @@ RUN site_packages="$(python -c "import sysconfig; print(sysconfig.get_paths()['p
 RUN python -c "from importlib.metadata import version as v; from packaging.version import Version as V; \
                assert V(v('aiohttp')) >= V('3.14.3'), v('aiohttp'); \
                assert V(v('msgpack')) >= V('1.2.1'), v('msgpack'); \
+               assert V(v('nltk')) >= V('3.10.3'), v('nltk'); \
                assert V(v('setuptools')) >= V('78.1.1'), v('setuptools'); \
-               print('aiohttp/msgpack/setuptools security floors OK')" && \
+               print('aiohttp/msgpack/nltk/setuptools security floors OK')" && \
     python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
     ! find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' | grep -q .
 
diff --git a/pyproject.toml b/pyproject.toml
index 7076b965b4..cbd0288421 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,6 +84,7 @@ override-dependencies = [
     # environment above the first fixed releases.
     "aiohttp>=3.14.3",
     "msgpack>=1.2.1",
+    "nltk>=3.10.3",
     "setuptools>=78.1.1",
 ]
 
diff --git a/requirements/stem.txt b/requirements/stem.txt
index 6e84f85dac..39a1a2d2f7 100644
--- a/requirements/stem.txt
+++ b/requirements/stem.txt
@@ -92,7 +92,7 @@ mygene
 myvariant
 networkx
 nibabel
-nltk
+nltk>=3.10.3  # fixes CVE-2026-79675, CVE-2026-71513, CVE-2026-72818, CVE-2026-78680
 nuclear
 num2words
 numba
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index f0d53f3bd0..93fc950f7e 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -34,6 +34,7 @@
                                so guarded by importorskip)
   * aiohttp          >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
+  * nltk             >=3.10.3 (fixes CVE-2026-79675 and related High findings)
   * setuptools       >=78.1.1 (fixes CVE-2025-47273)
 
 All tests are CPU-only, hermetic (no sandbox container, no live LLM endpoint, no
@@ -71,11 +72,12 @@ def test_datamodel_code_generator_imports_at_fixed_version():
     assert Version(version("datamodel-code-generator")) >= Version("0.64.0")
 
 
-def test_aiohttp_msgpack_and_setuptools_security_floors():
+def test_python_security_floors():
     from packaging.version import Version
 
     assert Version(version("aiohttp")) >= Version("3.14.3")
     assert Version(version("msgpack")) >= Version("1.2.1")
+    assert Version(version("nltk")) >= Version("3.10.3")
     assert Version(version("setuptools")) >= Version("78.1.1")
 
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 0bce7d3ede..1c05e7e91e 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -23,6 +23,7 @@
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
+  * nltk             -> >=3.10.3 (fixes CVE-2026-79675 and related High findings)
   * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
   * typer            -> >=0.16   (click 8.2 compatible)
   * click            -> pin removed from requirements/pipeline.txt
@@ -155,18 +156,23 @@ def dockerfile(self):
     def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
         assert "WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7" in dockerfile
         assert "WANDB_GO_GIT_VERSION=5.19.2" in dockerfile
+        assert "WANDB_GO_CRYPTO_VERSION=0.55.0" in dockerfile
+        assert "WANDB_GO_IMAGE_VERSION=0.45.0" in dockerfile
+        assert "WANDB_GRPC_VERSION=1.83.1" in dockerfile
 
     @pytest.mark.parametrize(
         "expected",
         [
             "FROM golang:1.26.6 AS wandb-core-builder",
-            'go get "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}"',
+            '"github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}"',
             "go mod vendor",
             "go.mod",
             "vendor/modules.txt",
             'go version -m /wandb-core | grep -F "go1.26.6"',
             "github\\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}",
-            "google\\.golang\\.org/grpc[[:space:]]+v1\\.82\\.1",
+            "golang\\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}",
+            "golang\\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}",
+            "google\\.golang\\.org/grpc[[:space:]]+v${WANDB_GRPC_VERSION}",
             "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",
         ],
     )
@@ -185,6 +191,7 @@ def test_uv_git_cache_is_removed_from_final_image(self, dockerfile):
     def test_final_image_asserts_python_security_floors(self, dockerfile):
         assert "V(v('aiohttp')) >= V('3.14.3')" in dockerfile
         assert "V(v('msgpack')) >= V('1.2.1')" in dockerfile
+        assert "V(v('nltk')) >= V('3.10.3')" in dockerfile
         assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
 
     def test_ray_private_aiohttp_and_uv_build_sbom_are_remediated(self, dockerfile):
@@ -235,7 +242,7 @@ def test_nemo_run_and_launcher_pins_untouched(self):
 
 
 class TestStemRequirements:
-    """requirements/stem.txt: lxml security floor."""
+    """requirements/stem.txt security floors."""
 
     def test_lxml_pin_fixes_ghsa_vfmq_68hx_4jfw(self):
         req, comment = _find_requirement(STEM_REQUIREMENTS, "lxml")
@@ -253,6 +260,13 @@ def test_lxml_is_not_unbounded_or_unpinned(self):
             if code_part == "lxml":
                 pytest.fail(f"lxml requirement has no version floor: {raw_line!r}")
 
+    def test_nltk_floor_fixes_current_critical_and_high_findings(self):
+        req, comment = _find_requirement(STEM_REQUIREMENTS, "nltk")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for nltk, got {req.specifier}"
+        assert Version(specs[">="]) >= Version("3.10.3")
+        assert "CVE-2026-79675" in comment
+
 
 class TestPyprojectUvOverrides:
     """pyproject.toml: [tool.uv].override-dependencies still relaxes the transitive pins
@@ -285,7 +299,7 @@ def test_urllib3_override_present(self, uv_overrides):
 
     @pytest.mark.parametrize(
         ("package", "minimum"),
-        [("aiohttp", "3.14.3"), ("msgpack", "1.2.1"), ("setuptools", "78.1.1")],
+        [("aiohttp", "3.14.3"), ("msgpack", "1.2.1"), ("nltk", "3.10.3"), ("setuptools", "78.1.1")],
     )
     def test_container_security_override_present(self, uv_overrides, package, minimum):
         assert package in uv_overrides

From 775d2084fca7a82dc776b64bd242f9cb91b668c5 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 13:51:43 -0400
Subject: [PATCH 15/30] fix(container): pin resolved W&B text module

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 5 ++++-
 tests/test_requirements_versions.py | 3 ++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index d9baf78488..c5a6cc14ec 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -7,12 +7,14 @@ ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7
 ARG WANDB_GO_GIT_VERSION=5.19.2
 ARG WANDB_GO_CRYPTO_VERSION=0.55.0
 ARG WANDB_GO_IMAGE_VERSION=0.45.0
+ARG WANDB_GO_TEXT_VERSION=0.41.0
 ARG WANDB_GRPC_VERSION=1.83.1
 FROM golang:1.26.6 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
 ARG WANDB_GO_GIT_VERSION
 ARG WANDB_GO_CRYPTO_VERSION
 ARG WANDB_GO_IMAGE_VERSION
+ARG WANDB_GO_TEXT_VERSION
 ARG WANDB_GRPC_VERSION
 RUN git init /src/wandb && \
     cd /src/wandb && \
@@ -26,6 +28,7 @@ RUN cd /src/wandb/core && \
         "github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}" \
         "golang.org/x/crypto@v${WANDB_GO_CRYPTO_VERSION}" \
         "golang.org/x/image@v${WANDB_GO_IMAGE_VERSION}" \
+        "golang.org/x/text@v${WANDB_GO_TEXT_VERSION}" \
         "google.golang.org/grpc@v${WANDB_GRPC_VERSION}" && \
     go mod vendor && \
     grep -E "^[[:space:]]*github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" go.mod && \
@@ -41,7 +44,7 @@ RUN cd /src/wandb/core && \
     go version -m /wandb-core | grep -E "golang\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v${WANDB_GRPC_VERSION}([[:space:]]|$)" && \
-    go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)"
+    go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v${WANDB_GO_TEXT_VERSION}([[:space:]]|$)"
 
 # using ubuntu instead of debian for easier apptainer installation on arm64
 FROM ubuntu:22.04
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 1c05e7e91e..0063ad238e 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -158,6 +158,7 @@ def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
         assert "WANDB_GO_GIT_VERSION=5.19.2" in dockerfile
         assert "WANDB_GO_CRYPTO_VERSION=0.55.0" in dockerfile
         assert "WANDB_GO_IMAGE_VERSION=0.45.0" in dockerfile
+        assert "WANDB_GO_TEXT_VERSION=0.41.0" in dockerfile
         assert "WANDB_GRPC_VERSION=1.83.1" in dockerfile
 
     @pytest.mark.parametrize(
@@ -173,7 +174,7 @@ def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
             "golang\\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}",
             "golang\\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}",
             "google\\.golang\\.org/grpc[[:space:]]+v${WANDB_GRPC_VERSION}",
-            "golang\\.org/x/text[[:space:]]+v0\\.40\\.0",
+            "golang\\.org/x/text[[:space:]]+v${WANDB_GO_TEXT_VERSION}",
         ],
     )
     def test_fixed_go_components_are_build_time_verified(self, dockerfile, expected):

From fb683b658b846543c364f9dd17853472c1f888ac Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 15:24:44 -0400
Subject: [PATCH 16/30] fix(container): remove non-runtime scan inputs

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 31 ++++++++++++++++++++---------
 pyproject.toml                      |  1 +
 tests/test_dependency_functional.py |  8 ++++++++
 tests/test_requirements_versions.py | 20 +++++++++++++++++--
 4 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index c5a6cc14ec..2705ac743f 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -63,7 +63,11 @@ RUN apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10"
+# uv's wheel carries a build-environment SBOM containing old msgpack/setuptools
+# records. Delete it in the installation layer so scanners cannot mistake those
+# build-only records for packages in the final runtime image.
+RUN pip install --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10" && \
+    find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' -prune -exec rm -rf {} +
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
@@ -85,10 +89,13 @@ RUN apt-get update && \
         gir1.2-packagekitglib-1.0 && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-# for ifeval benchmark
-# TODO: can we get just a single dir?
+# IFEval imports this module from the repository root. Retain only the benchmark
+# it executes; unrelated manifests elsewhere in google-research describe Julia
+# and Rust projects that are neither installed nor used by this image.
 RUN mkdir /opt/benchmarks
-RUN git clone https://github.com/google-research/google-research.git /opt/benchmarks/google-research --depth=1
+RUN git clone https://github.com/google-research/google-research.git /opt/benchmarks/google-research --depth=1 && \
+    cd /opt/benchmarks/google-research && \
+    find . -mindepth 1 -maxdepth 1 ! -name instruction_following_eval -exec rm -rf {} +
 
 RUN git clone https://github.com/ShishirPatil/gorilla.git /opt/gorilla
 RUN cd /opt/gorilla && git checkout 86d0374d0db52623c5092a73f82c22b87b7e9a25
@@ -137,8 +144,8 @@ RUN pip install ddgs
 # Ray's runtime-env agent prepends its bundled aiohttp 3.14.1 over the fixed
 # environment package. Replace that private copy with the resolved 3.14.3 files
 # from the same architecture (CVE-2026-69244). uv also packages a build SBOM
-# containing msgpack 1.1.2 and setuptools 70.3.0; remove that non-runtime SBOM so
-# scanners do not report uv's build dependencies as installed packages.
+# containing msgpack 1.1.2 and setuptools 70.3.0; its install-layer removal above
+# prevents scanners from reporting uv's build dependencies as installed packages.
 RUN site_packages="$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")" && \
     ray_thirdparty="${site_packages}/ray/_private/runtime_env/agent/thirdparty_files" && \
     rm -rf "${ray_thirdparty}/aiohttp" "${ray_thirdparty}"/aiohttp-*.dist-info && \
@@ -152,11 +159,17 @@ RUN python -c "from importlib.metadata import version as v; from packaging.versi
                assert V(v('aiohttp')) >= V('3.14.3'), v('aiohttp'); \
                assert V(v('msgpack')) >= V('1.2.1'), v('msgpack'); \
                assert V(v('nltk')) >= V('3.10.3'), v('nltk'); \
+               assert V(v('starlette')) >= V('1.3.1'), v('starlette'); \
                assert V(v('setuptools')) >= V('78.1.1'), v('setuptools'); \
-               print('aiohttp/msgpack/nltk/setuptools security floors OK')" && \
+               print('aiohttp/msgpack/nltk/starlette/setuptools security floors OK')" && \
+    python -c "from fastapi import FastAPI; FastAPI(); print('FastAPI/Starlette compatibility OK')" && \
     python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
     ! find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' | grep -q .
 
 # nSpect's global policy flags Git metadata left by uv's source-distribution
-# cache. The cache is build-only, so remove it from the published image.
-RUN rm -rf /root/.cache/uv
+# cache. Ray's Java runtime is not used by the Python jobs in this image, and
+# linux-libc-dev provides build-only headers. Remove all three after builds.
+RUN apt-get purge -y linux-libc-dev && \
+    rm -rf /root/.cache/uv /usr/local/lib/python3*/dist-packages/ray/jars && \
+    rm -rf /var/cache/apt/archives /var/lib/apt/lists/* && \
+    [ -z "$(find /usr/local/lib/python3*/dist-packages -path '*/ray/jars/*' -type f 2>/dev/null)" ]
diff --git a/pyproject.toml b/pyproject.toml
index cbd0288421..33b6d8feb6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -85,6 +85,7 @@ override-dependencies = [
     "aiohttp>=3.14.3",
     "msgpack>=1.2.1",
     "nltk>=3.10.3",
+    "starlette>=1.3.1",
     "setuptools>=78.1.1",
 ]
 
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 93fc950f7e..9e3d935302 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -35,6 +35,7 @@
   * aiohttp          >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
   * nltk             >=3.10.3 (fixes CVE-2026-79675 and related High findings)
+  * starlette        >=1.3.1  (fixes CVE-2026-48818 and CVE-2026-54283)
   * setuptools       >=78.1.1 (fixes CVE-2025-47273)
 
 All tests are CPU-only, hermetic (no sandbox container, no live LLM endpoint, no
@@ -78,9 +79,16 @@ def test_python_security_floors():
     assert Version(version("aiohttp")) >= Version("3.14.3")
     assert Version(version("msgpack")) >= Version("1.2.1")
     assert Version(version("nltk")) >= Version("3.10.3")
+    assert Version(version("starlette")) >= Version("1.3.1")
     assert Version(version("setuptools")) >= Version("78.1.1")
 
 
+def test_fastapi_constructs_with_fixed_starlette():
+    from fastapi import FastAPI
+
+    assert FastAPI() is not None
+
+
 # ---------------------------------------------------------------------------
 # litellm 1.84.10 — driven through nemo_skills.inference.model
 # ---------------------------------------------------------------------------
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 0063ad238e..1d4264abf2 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -24,6 +24,7 @@
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
   * nltk             -> >=3.10.3 (fixes CVE-2026-79675 and related High findings)
+  * starlette        -> >=1.3.1  (fixes CVE-2026-48818 and CVE-2026-54283)
   * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
   * typer            -> >=0.16   (click 8.2 compatible)
   * click            -> pin removed from requirements/pipeline.txt
@@ -187,12 +188,13 @@ def test_verified_binary_replaces_wandb_release_binary(self, dockerfile):
         assert 'grep -F "Commit SHA: ${WANDB_CORE_COMMIT}"' in dockerfile
 
     def test_uv_git_cache_is_removed_from_final_image(self, dockerfile):
-        assert "RUN rm -rf /root/.cache/uv" in dockerfile
+        assert "rm -rf /root/.cache/uv" in dockerfile
 
     def test_final_image_asserts_python_security_floors(self, dockerfile):
         assert "V(v('aiohttp')) >= V('3.14.3')" in dockerfile
         assert "V(v('msgpack')) >= V('1.2.1')" in dockerfile
         assert "V(v('nltk')) >= V('3.10.3')" in dockerfile
+        assert "V(v('starlette')) >= V('1.3.1')" in dockerfile
         assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
 
     def test_ray_private_aiohttp_and_uv_build_sbom_are_remediated(self, dockerfile):
@@ -300,7 +302,13 @@ def test_urllib3_override_present(self, uv_overrides):
 
     @pytest.mark.parametrize(
         ("package", "minimum"),
-        [("aiohttp", "3.14.3"), ("msgpack", "1.2.1"), ("nltk", "3.10.3"), ("setuptools", "78.1.1")],
+        [
+            ("aiohttp", "3.14.3"),
+            ("msgpack", "1.2.1"),
+            ("nltk", "3.10.3"),
+            ("starlette", "1.3.1"),
+            ("setuptools", "78.1.1"),
+        ],
     )
     def test_container_security_override_present(self, uv_overrides, package, minimum):
         assert package in uv_overrides
@@ -347,3 +355,11 @@ def test_setuptools_build_floor_fixes_cve_2025_47273(pyproject):
     specs = {spec.operator: spec.version for spec in requirement.specifier}
     assert ">=" in specs
     assert Version(specs[">="]) >= Version("78.1.1")
+
+
+def test_container_drops_non_runtime_scan_inputs():
+    dockerfile = NEMO_SKILLS_DOCKERFILE.read_text()
+    assert "! -name instruction_following_eval" in dockerfile
+    assert "apt-get purge -y linux-libc-dev" in dockerfile
+    assert "ray/jars" in dockerfile
+    assert dockerfile.index("uv-*.dist-info/sboms") > dockerfile.index("pip install --upgrade pip")

From ee1eb08863861739c2d71ff794e21f6214857f5b Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 15:37:09 -0400
Subject: [PATCH 17/30] fix(container): avoid caching uv build metadata

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 2 +-
 tests/test_requirements_versions.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 2705ac743f..87939345a2 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -66,7 +66,7 @@ RUN apt-get update && \
 # uv's wheel carries a build-environment SBOM containing old msgpack/setuptools
 # records. Delete it in the installation layer so scanners cannot mistake those
 # build-only records for packages in the final runtime image.
-RUN pip install --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10" && \
+RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10" && \
     find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' -prune -exec rm -rf {} +
 
 # Update package lists and install apptainer for arm64
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 1d4264abf2..ef56c1d019 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -362,4 +362,5 @@ def test_container_drops_non_runtime_scan_inputs():
     assert "! -name instruction_following_eval" in dockerfile
     assert "apt-get purge -y linux-libc-dev" in dockerfile
     assert "ray/jars" in dockerfile
-    assert dockerfile.index("uv-*.dist-info/sboms") > dockerfile.index("pip install --upgrade pip")
+    uv_bootstrap = dockerfile.index("pip install --no-cache-dir --upgrade pip")
+    assert dockerfile.index("uv-*.dist-info/sboms") > uv_bootstrap

From d3eef3e55ae1f372c068e53474cec05032e87b4f Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 15:53:03 -0400
Subject: [PATCH 18/30] fix(container): isolate uv build metadata

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 21 ++++++++++-----------
 tests/test_requirements_versions.py | 11 +++++++----
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 87939345a2..d4da337fcc 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -46,6 +46,10 @@ RUN cd /src/wandb/core && \
     go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v${WANDB_GRPC_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v${WANDB_GO_TEXT_VERSION}([[:space:]]|$)"
 
+# Keep uv's Python wheel and its build-environment SBOM out of the runtime
+# image. Only the standalone, version-pinned executable crosses this stage.
+FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer
+
 # using ubuntu instead of debian for easier apptainer installation on arm64
 FROM ubuntu:22.04
 ARG WANDB_CORE_COMMIT
@@ -63,11 +67,9 @@ RUN apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-# uv's wheel carries a build-environment SBOM containing old msgpack/setuptools
-# records. Delete it in the installation layer so scanners cannot mistake those
-# build-only records for packages in the final runtime image.
-RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "uv>=0.11.10" && \
-    find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' -prune -exec rm -rf {} +
+COPY --from=uv-installer /uv /usr/local/bin/uv
+RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" && \
+    uv --version | grep -Fx 'uv 0.12.9'
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
@@ -143,15 +145,12 @@ RUN pip install ddgs
 
 # Ray's runtime-env agent prepends its bundled aiohttp 3.14.1 over the fixed
 # environment package. Replace that private copy with the resolved 3.14.3 files
-# from the same architecture (CVE-2026-69244). uv also packages a build SBOM
-# containing msgpack 1.1.2 and setuptools 70.3.0; its install-layer removal above
-# prevents scanners from reporting uv's build dependencies as installed packages.
+# from the same architecture (CVE-2026-69244).
 RUN site_packages="$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")" && \
     ray_thirdparty="${site_packages}/ray/_private/runtime_env/agent/thirdparty_files" && \
     rm -rf "${ray_thirdparty}/aiohttp" "${ray_thirdparty}"/aiohttp-*.dist-info && \
     cp -a "${site_packages}/aiohttp" "${ray_thirdparty}/" && \
-    cp -a "${site_packages}"/aiohttp-*.dist-info "${ray_thirdparty}/" && \
-    find "${site_packages}" -type d -path '*/uv-*.dist-info/sboms' -prune -exec rm -rf {} +
+    cp -a "${site_packages}"/aiohttp-*.dist-info "${ray_thirdparty}/"
 
 # Guard the final resolved environment and Ray's private import path against the
 # High findings seen in the multi-architecture image scans.
@@ -164,7 +163,7 @@ RUN python -c "from importlib.metadata import version as v; from packaging.versi
                print('aiohttp/msgpack/nltk/starlette/setuptools security floors OK')" && \
     python -c "from fastapi import FastAPI; FastAPI(); print('FastAPI/Starlette compatibility OK')" && \
     python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
-    ! find /usr/local/lib/python3*/dist-packages -type d -path '*/uv-*.dist-info/sboms' | grep -q .
+    [ -z "$(find /usr/local/lib/python3*/dist-packages -maxdepth 1 -type d -name 'uv-*.dist-info' 2>/dev/null)" ]
 
 # nSpect's global policy flags Git metadata left by uv's source-distribution
 # cache. Ray's Java runtime is not used by the Python jobs in this image, and
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index ef56c1d019..629b2ec35a 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -197,9 +197,12 @@ def test_final_image_asserts_python_security_floors(self, dockerfile):
         assert "V(v('starlette')) >= V('1.3.1')" in dockerfile
         assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
 
-    def test_ray_private_aiohttp_and_uv_build_sbom_are_remediated(self, dockerfile):
+    def test_ray_private_aiohttp_and_uv_wheel_are_remediated(self, dockerfile):
         assert "ray/_private/runtime_env/agent/thirdparty_files" in dockerfile
-        assert "uv-*.dist-info/sboms" in dockerfile
+        assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
+        assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile
+        assert "uv --version | grep -Fx 'uv 0.12.9'" in dockerfile
+        assert '"uv>=0.11.10"' not in dockerfile
 
 
 class TestPipelineRequirements:
@@ -362,5 +365,5 @@ def test_container_drops_non_runtime_scan_inputs():
     assert "! -name instruction_following_eval" in dockerfile
     assert "apt-get purge -y linux-libc-dev" in dockerfile
     assert "ray/jars" in dockerfile
-    uv_bootstrap = dockerfile.index("pip install --no-cache-dir --upgrade pip")
-    assert dockerfile.index("uv-*.dist-info/sboms") > uv_bootstrap
+    assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
+    assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile

From 9ef38fc6b3ffbc14f7d78f0c1f5f9d3013f7121a Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 15:57:49 -0400
Subject: [PATCH 19/30] fix(container): accept uv build metadata

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 2 +-
 tests/test_requirements_versions.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index d4da337fcc..337d2d0dbe 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -69,7 +69,7 @@ RUN apt-get update && \
 
 COPY --from=uv-installer /uv /usr/local/bin/uv
 RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" && \
-    uv --version | grep -Fx 'uv 0.12.9'
+    uv --version | grep -E '^uv 0\.12\.9([[:space:]]|$)'
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 629b2ec35a..21aff09d52 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -201,7 +201,7 @@ def test_ray_private_aiohttp_and_uv_wheel_are_remediated(self, dockerfile):
         assert "ray/_private/runtime_env/agent/thirdparty_files" in dockerfile
         assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
         assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile
-        assert "uv --version | grep -Fx 'uv 0.12.9'" in dockerfile
+        assert "uv --version | grep -E '^uv 0\\.12\\.9([[:space:]]|$)'" in dockerfile
         assert '"uv>=0.11.10"' not in dockerfile
 
 

From 8c43f5547e374f440515a583f6233c3d1fec7a35 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 16:12:25 -0400
Subject: [PATCH 20/30] fix(container): exclude uv from runtime layers

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 21 +++++++++++++++------
 tests/test_requirements_versions.py |  4 ++++
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 337d2d0dbe..540d552288 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -51,7 +51,7 @@ RUN cd /src/wandb/core && \
 FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer
 
 # using ubuntu instead of debian for easier apptainer installation on arm64
-FROM ubuntu:22.04
+FROM ubuntu:22.04 AS runtime-base
 ARG WANDB_CORE_COMMIT
 
 # Install Python and other dependencies
@@ -67,9 +67,7 @@ RUN apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-COPY --from=uv-installer /uv /usr/local/bin/uv
-RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" && \
-    uv --version | grep -E '^uv 0\.12\.9([[:space:]]|$)'
+RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1"
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
@@ -133,8 +131,18 @@ ARG CACHEBUST=4
 # Install via `uv pip` from the project directory so [tool.uv].override-dependencies
 # in pyproject.toml (which relaxes leptonai's httpx==0.27.2 pin so litellm 1.84.x
 # can be installed) is picked up. Plain pip ignores [tool.uv] and the resolver fails.
+FROM runtime-base AS dependency-installer
+COPY --from=uv-installer /uv /usr/local/bin/uv
+RUN uv --version | grep -E '^uv 0\.12\.9([[:space:]]|$)'
 RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \
-    -r core/requirements.txt -r requirements/pipeline.txt
+    -r core/requirements.txt -r requirements/pipeline.txt && \
+    rm -f /usr/local/bin/uv && rm -rf /root/.cache/uv
+
+# Copy the resolved environment, not the uv layer that created it. This keeps
+# uv's embedded build SBOM out of the final image while retaining all wheels,
+# console scripts, and package data under /usr/local.
+FROM runtime-base
+COPY --from=dependency-installer /usr/local/ /usr/local/
 # Replace W&B's vulnerable release binary with the source-compatible patched core
 # built and module-verified above. The copy preserves its executable mode.
 COPY --from=wandb-core-builder /wandb-core /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core
@@ -163,12 +171,13 @@ RUN python -c "from importlib.metadata import version as v; from packaging.versi
                print('aiohttp/msgpack/nltk/starlette/setuptools security floors OK')" && \
     python -c "from fastapi import FastAPI; FastAPI(); print('FastAPI/Starlette compatibility OK')" && \
     python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
+    [ ! -e /usr/local/bin/uv ] && \
     [ -z "$(find /usr/local/lib/python3*/dist-packages -maxdepth 1 -type d -name 'uv-*.dist-info' 2>/dev/null)" ]
 
 # nSpect's global policy flags Git metadata left by uv's source-distribution
 # cache. Ray's Java runtime is not used by the Python jobs in this image, and
 # linux-libc-dev provides build-only headers. Remove all three after builds.
 RUN apt-get purge -y linux-libc-dev && \
-    rm -rf /root/.cache/uv /usr/local/lib/python3*/dist-packages/ray/jars && \
+    rm -rf /usr/local/lib/python3*/dist-packages/ray/jars && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/* && \
     [ -z "$(find /usr/local/lib/python3*/dist-packages -path '*/ray/jars/*' -type f 2>/dev/null)" ]
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 21aff09d52..dae4e540f3 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -202,6 +202,9 @@ def test_ray_private_aiohttp_and_uv_wheel_are_remediated(self, dockerfile):
         assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
         assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile
         assert "uv --version | grep -E '^uv 0\\.12\\.9([[:space:]]|$)'" in dockerfile
+        assert "FROM runtime-base AS dependency-installer" in dockerfile
+        assert "rm -f /usr/local/bin/uv" in dockerfile
+        assert "COPY --from=dependency-installer /usr/local/ /usr/local/" in dockerfile
         assert '"uv>=0.11.10"' not in dockerfile
 
 
@@ -367,3 +370,4 @@ def test_container_drops_non_runtime_scan_inputs():
     assert "ray/jars" in dockerfile
     assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
     assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile
+    assert "COPY --from=dependency-installer /usr/local/ /usr/local/" in dockerfile

From baee6bd4c7015388e97a753ec8614063d7f6ac88 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Thu, 3 Sep 2026 16:27:41 -0400
Subject: [PATCH 21/30] fix(container): drop pip build BOM

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 11 +++++++----
 tests/test_requirements_versions.py |  3 +++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index 540d552288..cd0a3175d4 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -67,7 +67,8 @@ RUN apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
-RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1"
+RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" && \
+    rm -f /usr/local/lib/python3*/dist-packages/pip/_vendor/bom.cdx.json
 
 # Update package lists and install apptainer for arm64
 # https://apptainer.org/docs/admin/1.1/installation.html
@@ -138,10 +139,11 @@ RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \
     -r core/requirements.txt -r requirements/pipeline.txt && \
     rm -f /usr/local/bin/uv && rm -rf /root/.cache/uv
 
-# Copy the resolved environment, not the uv layer that created it. This keeps
-# uv's embedded build SBOM out of the final image while retaining all wheels,
-# console scripts, and package data under /usr/local.
+# Replace /usr/local with the resolved environment, not the uv layer that
+# created it. Replacement prevents metadata from older pre-resolution packages
+# from surviving alongside the resolved versions.
 FROM runtime-base
+RUN rm -rf /usr/local
 COPY --from=dependency-installer /usr/local/ /usr/local/
 # Replace W&B's vulnerable release binary with the source-compatible patched core
 # built and module-verified above. The copy preserves its executable mode.
@@ -172,6 +174,7 @@ RUN python -c "from importlib.metadata import version as v; from packaging.versi
     python -c "from fastapi import FastAPI; FastAPI(); print('FastAPI/Starlette compatibility OK')" && \
     python -c "import sys; from importlib.util import find_spec; from pathlib import Path; from packaging.version import Version; thirdparty = Path(find_spec('ray').submodule_search_locations[0]) / '_private/runtime_env/agent/thirdparty_files'; sys.path.insert(0, str(thirdparty)); import aiohttp, ray; assert Path(aiohttp.__file__).is_relative_to(thirdparty), aiohttp.__file__; assert Version(aiohttp.__version__) >= Version('3.14.3'), aiohttp.__version__; print('Ray private aiohttp OK:', ray.__version__, aiohttp.__version__)" && \
     [ ! -e /usr/local/bin/uv ] && \
+    [ ! -e /usr/local/lib/python3.10/dist-packages/pip/_vendor/bom.cdx.json ] && \
     [ -z "$(find /usr/local/lib/python3*/dist-packages -maxdepth 1 -type d -name 'uv-*.dist-info' 2>/dev/null)" ]
 
 # nSpect's global policy flags Git metadata left by uv's source-distribution
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index dae4e540f3..299ae19319 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -204,6 +204,8 @@ def test_ray_private_aiohttp_and_uv_wheel_are_remediated(self, dockerfile):
         assert "uv --version | grep -E '^uv 0\\.12\\.9([[:space:]]|$)'" in dockerfile
         assert "FROM runtime-base AS dependency-installer" in dockerfile
         assert "rm -f /usr/local/bin/uv" in dockerfile
+        assert "rm -f /usr/local/lib/python3*/dist-packages/pip/_vendor/bom.cdx.json" in dockerfile
+        assert "RUN rm -rf /usr/local" in dockerfile
         assert "COPY --from=dependency-installer /usr/local/ /usr/local/" in dockerfile
         assert '"uv>=0.11.10"' not in dockerfile
 
@@ -370,4 +372,5 @@ def test_container_drops_non_runtime_scan_inputs():
     assert "ray/jars" in dockerfile
     assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
     assert "COPY --from=uv-installer /uv /usr/local/bin/uv" in dockerfile
+    assert "RUN rm -rf /usr/local" in dockerfile
     assert "COPY --from=dependency-installer /usr/local/ /usr/local/" in dockerfile

From c315300244c563ec6f14bcbb73cd9d59364e6905 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 4 Sep 2026 09:05:20 -0400
Subject: [PATCH 22/30] fix(container): cross-compile W&B core for target arch

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 dockerfiles/Dockerfile.nemo-skills  | 7 +++++--
 tests/test_requirements_versions.py | 6 +++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index cd0a3175d4..db97aeac0f 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -9,13 +9,14 @@ ARG WANDB_GO_CRYPTO_VERSION=0.55.0
 ARG WANDB_GO_IMAGE_VERSION=0.45.0
 ARG WANDB_GO_TEXT_VERSION=0.41.0
 ARG WANDB_GRPC_VERSION=1.83.1
-FROM golang:1.26.6 AS wandb-core-builder
+FROM --platform=$BUILDPLATFORM golang:1.26.6 AS wandb-core-builder
 ARG WANDB_CORE_COMMIT
 ARG WANDB_GO_GIT_VERSION
 ARG WANDB_GO_CRYPTO_VERSION
 ARG WANDB_GO_IMAGE_VERSION
 ARG WANDB_GO_TEXT_VERSION
 ARG WANDB_GRPC_VERSION
+ARG TARGETARCH
 RUN git init /src/wandb && \
     cd /src/wandb && \
     git remote add origin https://github.com/wandb/wandb.git && \
@@ -33,13 +34,15 @@ RUN cd /src/wandb/core && \
     go mod vendor && \
     grep -E "^[[:space:]]*github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" go.mod && \
     grep -E "^# github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" vendor/modules.txt && \
-    CGO_ENABLED=0 go build \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
         -tags "disable_grpc_modules parquet_read_only" \
         -ldflags "-s -w -X main.commit=${WANDB_CORE_COMMIT}" \
         -mod=vendor \
         -o /wandb-core \
         ./cmd/wandb-core && \
     go version -m /wandb-core | grep -F "go1.26.6" && \
+    go version -m /wandb-core | grep -E "build[[:space:]]+GOOS=linux([[:space:]]|$)" && \
+    go version -m /wandb-core | grep -E "build[[:space:]]+GOARCH=${TARGETARCH}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "github\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}([[:space:]]|$)" && \
     go version -m /wandb-core | grep -E "golang\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}([[:space:]]|$)" && \
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index 299ae19319..c98f545f67 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -165,12 +165,16 @@ def test_immutable_upstream_security_commit_is_pinned(self, dockerfile):
     @pytest.mark.parametrize(
         "expected",
         [
-            "FROM golang:1.26.6 AS wandb-core-builder",
+            "FROM --platform=$BUILDPLATFORM golang:1.26.6 AS wandb-core-builder",
+            "ARG TARGETARCH",
             '"github.com/go-git/go-git/v5@v${WANDB_GO_GIT_VERSION}"',
             "go mod vendor",
             "go.mod",
             "vendor/modules.txt",
             'go version -m /wandb-core | grep -F "go1.26.6"',
+            "CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build",
+            "build[[:space:]]+GOOS=linux([[:space:]]|$)",
+            "build[[:space:]]+GOARCH=${TARGETARCH}([[:space:]]|$)",
             "github\\.com/go-git/go-git/v5[[:space:]]+v${WANDB_GO_GIT_VERSION}",
             "golang\\.org/x/crypto[[:space:]]+v${WANDB_GO_CRYPTO_VERSION}",
             "golang\\.org/x/image[[:space:]]+v${WANDB_GO_IMAGE_VERSION}",

From da8459af101ce32f7c9ffd6cf6f0b27b40e65fc3 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 11 Sep 2026 14:36:22 -0400
Subject: [PATCH 23/30] test: replace retired NVIDIA API model

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 docs/evaluation/external-benchmarks.md |  5 +++--
 tests/test_eval.py                     |  5 +++--
 tests/test_generation.py               | 25 +++++++++++++++----------
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/docs/evaluation/external-benchmarks.md b/docs/evaluation/external-benchmarks.md
index a4c4ca4e8b..7f0acb994d 100644
--- a/docs/evaluation/external-benchmarks.md
+++ b/docs/evaluation/external-benchmarks.md
@@ -307,10 +307,11 @@ Run evaluation (using an API model as an example):
 ns eval \
     --cluster=local \
     --server_type=openai \
-    --model=nvidia/nemotron-3-nano-30b-a3b \
+    --model=nvidia/nemotron-3.5-lightning-30b-a3b \
     --server_address=https://integrate.api.nvidia.com/v1 \
     --benchmarks=word_count \
-    --output_dir=/workspace/test-eval
+    --output_dir=/workspace/test-eval \
+    ++inference.temperature=1.0
 ```
 
 View results:
diff --git a/tests/test_eval.py b/tests/test_eval.py
index 272728fca3..15474a6e9b 100644
--- a/tests/test_eval.py
+++ b/tests/test_eval.py
@@ -288,8 +288,8 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"ns eval "
         f"    --server_type=openai "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
@@ -297,6 +297,7 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"    --generation_module={shlex.quote(str(generation_module))} "
         f"    ++max_samples=1 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
diff --git a/tests/test_generation.py b/tests/test_generation.py
index 297d0b180f..be1bd60705 100644
--- a/tests/test_generation.py
+++ b/tests/test_generation.py
@@ -33,12 +33,13 @@ def test_eval_gsm8k_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -64,17 +65,18 @@ def test_eval_judge_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=math-500 "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --judge_model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --judge_generation_type=math_judge "
-        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1' "
+        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1' "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -100,7 +102,7 @@ def test_fail_on_api_key_env_var(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
@@ -122,12 +124,13 @@ def test_succeed_on_api_key_env_var(tmp_path):
         f"unset NVIDIA_API_KEY && "
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++server.api_key_env_var=MY_CUSTOM_KEY "
@@ -155,12 +158,13 @@ def test_generate_openai_format(tmp_path, format):
     cmd = (
         f"ns generate "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --input_file=/nemo_run/code/tests/data/openai-input-{format}.test "
         f"    --output_dir={tmp_path} "
         f"    ++prompt_format=openai "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -370,17 +374,18 @@ def test_judge_generations_with_structured_output(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=hle "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3-nano-30b-a3b "
+        f"    --judge_model=nvidia/nemotron-3.5-lightning-30b-a3b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --metric_type=hle-aa "
-        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1" '
+        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1" '
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++inference.tokens_to_generate=1024 "  # to make test go fast

From ef58ed3b4f5feeaef3e5c822716688ffaffc8c84 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 11 Sep 2026 15:30:56 -0400
Subject: [PATCH 24/30] test: use accessible NVIDIA API model

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 docs/evaluation/external-benchmarks.md |  5 ++---
 tests/test_eval.py                     |  5 ++---
 tests/test_generation.py               | 25 ++++++++++---------------
 3 files changed, 14 insertions(+), 21 deletions(-)

diff --git a/docs/evaluation/external-benchmarks.md b/docs/evaluation/external-benchmarks.md
index 7f0acb994d..0ecc999339 100644
--- a/docs/evaluation/external-benchmarks.md
+++ b/docs/evaluation/external-benchmarks.md
@@ -307,11 +307,10 @@ Run evaluation (using an API model as an example):
 ns eval \
     --cluster=local \
     --server_type=openai \
-    --model=nvidia/nemotron-3.5-lightning-30b-a3b \
+    --model=nvidia/nemotron-3-super-120b-a12b \
     --server_address=https://integrate.api.nvidia.com/v1 \
     --benchmarks=word_count \
-    --output_dir=/workspace/test-eval \
-    ++inference.temperature=1.0
+    --output_dir=/workspace/test-eval
 ```
 
 View results:
diff --git a/tests/test_eval.py b/tests/test_eval.py
index 15474a6e9b..b2791ea08b 100644
--- a/tests/test_eval.py
+++ b/tests/test_eval.py
@@ -288,8 +288,8 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"ns eval "
         f"    --server_type=openai "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
@@ -297,7 +297,6 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"    --generation_module={shlex.quote(str(generation_module))} "
         f"    ++max_samples=1 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
diff --git a/tests/test_generation.py b/tests/test_generation.py
index be1bd60705..7f07845b10 100644
--- a/tests/test_generation.py
+++ b/tests/test_generation.py
@@ -33,13 +33,12 @@ def test_eval_gsm8k_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -65,18 +64,17 @@ def test_eval_judge_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=math-500 "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --judge_model=nvidia/nemotron-3-super-120b-a12b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --judge_generation_type=math_judge "
-        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1' "
+        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1' "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -102,7 +100,7 @@ def test_fail_on_api_key_env_var(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
@@ -124,13 +122,12 @@ def test_succeed_on_api_key_env_var(tmp_path):
         f"unset NVIDIA_API_KEY && "
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++server.api_key_env_var=MY_CUSTOM_KEY "
@@ -158,13 +155,12 @@ def test_generate_openai_format(tmp_path, format):
     cmd = (
         f"ns generate "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --input_file=/nemo_run/code/tests/data/openai-input-{format}.test "
         f"    --output_dir={tmp_path} "
         f"    ++prompt_format=openai "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -374,18 +370,17 @@ def test_judge_generations_with_structured_output(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --model=nvidia/nemotron-3-super-120b-a12b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=hle "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3.5-lightning-30b-a3b "
+        f"    --judge_model=nvidia/nemotron-3-super-120b-a12b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --metric_type=hle-aa "
-        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1" '
+        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1" '
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++inference.tokens_to_generate=1024 "  # to make test go fast

From 59dadf5c1ee3684cfcaeef13e40b520a7ac2c6d5 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 11 Sep 2026 15:33:25 -0400
Subject: [PATCH 25/30] test: use durable NVIDIA API endpoint

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 docs/evaluation/external-benchmarks.md |  5 +++--
 tests/test_eval.py                     |  5 +++--
 tests/test_generation.py               | 25 +++++++++++++++----------
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/docs/evaluation/external-benchmarks.md b/docs/evaluation/external-benchmarks.md
index 0ecc999339..335420870c 100644
--- a/docs/evaluation/external-benchmarks.md
+++ b/docs/evaluation/external-benchmarks.md
@@ -307,10 +307,11 @@ Run evaluation (using an API model as an example):
 ns eval \
     --cluster=local \
     --server_type=openai \
-    --model=nvidia/nemotron-3-super-120b-a12b \
+    --model=openai/gpt-oss-20b \
     --server_address=https://integrate.api.nvidia.com/v1 \
     --benchmarks=word_count \
-    --output_dir=/workspace/test-eval
+    --output_dir=/workspace/test-eval \
+    ++inference.temperature=1.0
 ```
 
 View results:
diff --git a/tests/test_eval.py b/tests/test_eval.py
index b2791ea08b..29668b03c5 100644
--- a/tests/test_eval.py
+++ b/tests/test_eval.py
@@ -288,8 +288,8 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"ns eval "
         f"    --server_type=openai "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
@@ -297,6 +297,7 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"    --generation_module={shlex.quote(str(generation_module))} "
         f"    ++max_samples=1 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
diff --git a/tests/test_generation.py b/tests/test_generation.py
index 7f07845b10..98ec82ef6b 100644
--- a/tests/test_generation.py
+++ b/tests/test_generation.py
@@ -33,12 +33,13 @@ def test_eval_gsm8k_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -64,17 +65,18 @@ def test_eval_judge_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=math-500 "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --judge_model=openai/gpt-oss-20b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --judge_generation_type=math_judge "
-        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1' "
+        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1' "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -100,7 +102,7 @@ def test_fail_on_api_key_env_var(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
@@ -122,12 +124,13 @@ def test_succeed_on_api_key_env_var(tmp_path):
         f"unset NVIDIA_API_KEY && "
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++server.api_key_env_var=MY_CUSTOM_KEY "
@@ -155,12 +158,13 @@ def test_generate_openai_format(tmp_path, format):
     cmd = (
         f"ns generate "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --input_file=/nemo_run/code/tests/data/openai-input-{format}.test "
         f"    --output_dir={tmp_path} "
         f"    ++prompt_format=openai "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
     )
@@ -370,17 +374,18 @@ def test_judge_generations_with_structured_output(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --model=openai/gpt-oss-20b "
         f"    --server_address=https://integrate.api.nvidia.com/v1 "
         f"    --benchmarks=hle "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=nvidia/nemotron-3-super-120b-a12b "
+        f"    --judge_model=openai/gpt-oss-20b "
         f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
         f"    --judge_server_type=openai "
         f"    --metric_type=hle-aa "
-        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1" '
+        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1" '
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
+        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++inference.tokens_to_generate=1024 "  # to make test go fast

From 87367bb1cf1e305e22cb8c5d238004bd67da4113 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 11 Sep 2026 16:11:20 -0400
Subject: [PATCH 26/30] test: use entitled NVIDIA API endpoint

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 tests/test_eval.py       | 19 +++++++++-----
 tests/test_generation.py | 53 +++++++++++++++++++++-------------------
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/tests/test_eval.py b/tests/test_eval.py
index 29668b03c5..95a1c1de7a 100644
--- a/tests/test_eval.py
+++ b/tests/test_eval.py
@@ -27,6 +27,10 @@
 from nemo_skills.pipeline.utils import eval as eval_utils
 from nemo_skills.pipeline.utils.scripts import BaseJobScript, EvalClientScript
 
+NVIDIA_TEST_API_BASE_URL = "https://inference-api.nvidia.com/v1"
+NVIDIA_TEST_API_KEY_ENV_VAR = "NV_INFERENCE_API_KEY"
+NVIDIA_TEST_API_MODEL = "gcp/google/gemini-2.5-flash-lite"
+
 
 class FakeExp:
     def __enter__(self):
@@ -278,7 +282,10 @@ def fake_generate(**kwargs):
 
 
 @pytest.mark.timeout(300)
-@pytest.mark.skipif("NVIDIA_API_KEY" not in os.environ, reason="requires NVIDIA_API_KEY")
+@pytest.mark.skipif(
+    NVIDIA_TEST_API_KEY_ENV_VAR not in os.environ,
+    reason=f"requires {NVIDIA_TEST_API_KEY_ENV_VAR}",
+)
 def test_eval_multi_model_generation_module_smoke(tmp_path):
     repo_root = Path(__file__).resolve().parents[1]
     output_dir = tmp_path / "out"
@@ -288,18 +295,18 @@ def test_eval_multi_model_generation_module_smoke(tmp_path):
         f"ns eval "
         f"    --server_type=openai "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={shlex.quote(str(output_dir))} "
         f"    --generation_module={shlex.quote(str(generation_module))} "
         f"    ++max_samples=1 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
+        f"    ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR} "
     )
     env = {**os.environ, "PYTHONPATH": f"{repo_root}{os.pathsep}{os.environ.get('PYTHONPATH', '')}"}
     subprocess.run(cmd, shell=True, check=True, env=env)
diff --git a/tests/test_generation.py b/tests/test_generation.py
index 98ec82ef6b..55f3036df7 100644
--- a/tests/test_generation.py
+++ b/tests/test_generation.py
@@ -27,21 +27,25 @@
 from nemo_skills.pipeline.utils.generation import configure_client
 from nemo_skills.pipeline.utils.scripts import ServerScript
 
+NVIDIA_TEST_API_BASE_URL = "https://inference-api.nvidia.com/v1"
+NVIDIA_TEST_API_KEY_ENV_VAR = "NV_INFERENCE_API_KEY"
+NVIDIA_TEST_API_MODEL = "gcp/google/gemini-2.5-flash-lite"
+
 
 @pytest.mark.timeout(300)
 def test_eval_gsm8k_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
+        f"    ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR} "
     )
     subprocess.run(cmd, shell=True, check=True)
 
@@ -65,20 +69,20 @@ def test_eval_judge_api(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=math-500 "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=openai/gpt-oss-20b "
-        f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --judge_model={NVIDIA_TEST_API_MODEL} "
+        f"    --judge_server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --judge_server_type=openai "
         f"    --judge_generation_type=math_judge "
-        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1' "
+        f"    --extra_judge_args='++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1 ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR}' "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
+        f"    ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR} "
     )
     subprocess.run(cmd, shell=True, check=True)
 
@@ -102,8 +106,8 @@ def test_fail_on_api_key_env_var(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
@@ -120,17 +124,16 @@ def test_fail_on_api_key_env_var(tmp_path):
 @pytest.mark.timeout(300)
 def test_succeed_on_api_key_env_var(tmp_path):
     cmd = (
-        f"export MY_CUSTOM_KEY=$NVIDIA_API_KEY && "
-        f"unset NVIDIA_API_KEY && "
+        f"export MY_CUSTOM_KEY=${NVIDIA_TEST_API_KEY_ENV_VAR} && "
+        f"unset NVIDIA_API_KEY {NVIDIA_TEST_API_KEY_ENV_VAR} && "
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=gsm8k "
         f"    --output_dir={tmp_path} "
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
         f"    ++server.api_key_env_var=MY_CUSTOM_KEY "
@@ -158,15 +161,15 @@ def test_generate_openai_format(tmp_path, format):
     cmd = (
         f"ns generate "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --input_file=/nemo_run/code/tests/data/openai-input-{format}.test "
         f"    --output_dir={tmp_path} "
         f"    ++prompt_format=openai "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
+        f"    ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR} "
     )
     subprocess.run(cmd, shell=True, check=True)
 
@@ -374,20 +377,20 @@ def test_judge_generations_with_structured_output(tmp_path):
     cmd = (
         f"ns eval "
         f"    --server_type=openai "
-        f"    --model=openai/gpt-oss-20b "
-        f"    --server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --model={NVIDIA_TEST_API_MODEL} "
+        f"    --server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --benchmarks=hle "
         f"    --output_dir={tmp_path} "
-        f"    --judge_model=openai/gpt-oss-20b "
-        f"    --judge_server_address=https://integrate.api.nvidia.com/v1 "
+        f"    --judge_model={NVIDIA_TEST_API_MODEL} "
+        f"    --judge_server_address={NVIDIA_TEST_API_BASE_URL} "
         f"    --judge_server_type=openai "
         f"    --metric_type=hle-aa "
-        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.temperature=1.0 ++inference.timeout=120 ++server.max_retries=1" '
+        f'    --extra_judge_args="++structured_output=HLE_JUDGE_AA ++max_concurrent_requests=1 ++inference.timeout=120 ++server.max_retries=1 ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR}" '
         f"    ++max_samples=2 "
         f"    ++max_concurrent_requests=1 "
-        f"    ++inference.temperature=1.0 "
         f"    ++inference.timeout=120 "
         f"    ++server.max_retries=1 "
+        f"    ++server.api_key_env_var={NVIDIA_TEST_API_KEY_ENV_VAR} "
         f"    ++inference.tokens_to_generate=1024 "  # to make test go fast
     )
     subprocess.run(cmd, shell=True, check=True)

From b7e957b5b9702949ae7c0a05fea7b09ad0ee83c6 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Fri, 2 Oct 2026 16:02:42 -0400
Subject: [PATCH 27/30] fix(security): raise transitive web dependency floors

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 pyproject.toml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 33b6d8feb6..260a8a3505 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,15 +80,28 @@ override-dependencies = [
     # 0.7.0 (a transitive dep of nemo_run) pins urllib3<1.27, but in practice
     # urllib3>=2 works at runtime, so override the constraint.
     "urllib3>=2.6.3",
+    # LeptonAI 0.27.3 caps AnyIO at <=4.9.0. Override that cap for the
+    # certificate-validation fix in CVE-2026-63374.
+    "anyio>=4.14.2",
+    # LeptonAI 0.27.3 pins instrumentator==7.0.0, which requires Starlette<1.0.
+    # Override it with the first release compatible with Starlette 1.x.
+    "prometheus-fastapi-instrumentator>=8.1.0",
     # Container scans found vulnerable transitive versions. Keep the complete
     # environment above the first fixed releases.
     "aiohttp>=3.14.3",
     "msgpack>=1.2.1",
     "nltk>=3.10.3",
-    "starlette>=1.3.1",
     "setuptools>=78.1.1",
 ]
 
+constraint-dependencies = [
+    # Fix CVE-2026-84381/CVE-2026-84382 in the HTTPX2 stack and
+    # CVE-2026-48818/CVE-2026-54283 in Starlette.
+    "httpcore2>=2.10.0",
+    "httpx2>=2.12.0",
+    "starlette>=1.3.1",
+]
+
 [tool.pytest.ini_options]
 markers = [
     "gpu: tests that require a GPU to run",

From 5eb62de2c06f9c3b0dafe3e094b0d27e6105a043 Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Tue, 6 Oct 2026 10:56:48 -0400
Subject: [PATCH 28/30] fix(security): pin gradio and repair policy tests

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/requirements.txt               |  2 +-
 tests/test_requirements_versions.py | 40 ++++++++++++++++++++++++++---
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/core/requirements.txt b/core/requirements.txt
index 1d3f3ac7c2..09234df9fc 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -17,7 +17,7 @@ flask
 func-timeout
 # Fixes all High GitPython advisories reported through GHSA-wvpp-8hx9-p66j.
 GitPython>=3.1.58
-gradio
+gradio>=6.16.0  # CVE-2026-49119
 httpx
 huggingface_hub
 hydra-core
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index c98f545f67..c657ca1bcc 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -24,6 +24,7 @@
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
   * nltk             -> >=3.10.3 (fixes CVE-2026-79675 and related High findings)
+  * gradio           -> >=6.16.0 (fixes CVE-2026-49119)
   * starlette        -> >=1.3.1  (fixes CVE-2026-48818 and CVE-2026-54283)
   * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
   * typer            -> >=0.16   (click 8.2 compatible)
@@ -130,6 +131,13 @@ def test_datamodel_code_generator_floor_fixes_all_eight_high_findings(self):
         assert ">=" in specs, f"expected a floor (>=) specifier for datamodel-code-generator, got {req.specifier}"
         assert Version(specs[">="]) >= Version("0.64.0")
 
+    def test_gradio_floor_fixes_cve_2026_49119(self):
+        req, comment = _find_requirement(CORE_REQUIREMENTS, "gradio")
+        specs = {spec.operator: spec.version for spec in req.specifier}
+        assert ">=" in specs, f"expected a floor (>=) specifier for gradio, got {req.specifier}"
+        assert Version(specs[">="]) >= Version("6.16.0")
+        assert "CVE-2026-49119" in comment
+
     def test_bfcl_does_not_reinstall_vulnerable_datamodel_code_generator(self):
         content = BFCL_MODULE.read_text()
         assert '"datamodel-code-generator==0.64.0"' in content
@@ -283,9 +291,8 @@ def test_nltk_floor_fixes_current_critical_and_high_findings(self):
         assert "CVE-2026-79675" in comment
 
 
-class TestPyprojectUvOverrides:
-    """pyproject.toml: [tool.uv].override-dependencies still relaxes the transitive pins
-    that would otherwise conflict with the new litellm floor."""
+class TestPyprojectUvDependencyPolicy:
+    """pyproject.toml keeps transitive security floors in the correct uv policy sections."""
 
     @pytest.fixture(scope="class")
     def uv_overrides(self):
@@ -297,6 +304,16 @@ def uv_overrides(self):
             parsed[req.name.lower()] = req
         return parsed
 
+    @pytest.fixture(scope="class")
+    def uv_constraints(self):
+        data = _load_toml(PYPROJECT_TOML)
+        constraints = data["tool"]["uv"]["constraint-dependencies"]
+        parsed = {}
+        for entry in constraints:
+            req = Requirement(entry)
+            parsed[req.name.lower()] = req
+        return parsed
+
     def test_httpx_override_present_for_litellm_compat(self, uv_overrides):
         assert "httpx" in uv_overrides, "expected an httpx override in [tool.uv].override-dependencies"
         req = uv_overrides["httpx"]
@@ -316,9 +333,10 @@ def test_urllib3_override_present(self, uv_overrides):
         ("package", "minimum"),
         [
             ("aiohttp", "3.14.3"),
+            ("anyio", "4.14.2"),
             ("msgpack", "1.2.1"),
             ("nltk", "3.10.3"),
-            ("starlette", "1.3.1"),
+            ("prometheus-fastapi-instrumentator", "8.1.0"),
             ("setuptools", "78.1.1"),
         ],
     )
@@ -328,6 +346,20 @@ def test_container_security_override_present(self, uv_overrides, package, minimu
         assert ">=" in specs
         assert Version(specs[">="]) >= Version(minimum)
 
+    @pytest.mark.parametrize(
+        ("package", "minimum"),
+        [
+            ("httpcore2", "2.10.0"),
+            ("httpx2", "2.12.0"),
+            ("starlette", "1.3.1"),
+        ],
+    )
+    def test_container_security_constraint_present(self, uv_constraints, package, minimum):
+        assert package in uv_constraints
+        specs = {spec.operator: spec.version for spec in uv_constraints[package].specifier}
+        assert ">=" in specs
+        assert Version(specs[">="]) >= Version(minimum)
+
     def test_dependencies_still_sourced_from_core_and_pipeline_requirements(self):
         data = _load_toml(PYPROJECT_TOML)
         dynamic_deps = data["tool"]["setuptools"]["dynamic"]["dependencies"]

From 1b61b89cf323354f1d6efa1dff25c69bdc5cbb8c Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Tue, 6 Oct 2026 13:28:14 -0400
Subject: [PATCH 29/30] fix(security): backport NLTK path containment

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 .gitignore                                    |  1 +
 core/requirements.txt                         |  3 +
 dockerfiles/Dockerfile.nemo-skills            | 10 +-
 dockerfiles/Dockerfile.sandbox                |  1 +
 pyproject.toml                                |  4 +-
 recipes/data-integrity/README.md              |  3 +-
 .../model_comparison/requirements.txt         |  2 +-
 requirements/sandbox.lock                     | 37 ++++---
 requirements/stem.txt                         |  2 +-
 security/README.md                            | 22 +++++
 security/nltk-cve-2026-81726.openvex.json     | 26 +++++
 tests/test_dependency_functional.py           | 98 ++++++++++++++++++-
 tests/test_requirements_versions.py           | 68 +++++++++++--
 13 files changed, 249 insertions(+), 28 deletions(-)
 create mode 100644 security/README.md
 create mode 100644 security/nltk-cve-2026-81726.openvex.json

diff --git a/.gitignore b/.gitignore
index 16e270f714..556006a3b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@
 
 *.json
 !greptile.json
+!security/nltk-cve-2026-81726.openvex.json
 !tests/data/dummy_external_benchmark/benchmark_map.json
 !nemo_skills/mcp/servers/exclude_domains_hle_opus.json
 *.tar.gz
diff --git a/core/requirements.txt b/core/requirements.txt
index 09234df9fc..b754ce9bf7 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -24,6 +24,9 @@ hydra-core
 ipython
 iso639-lang
 koifeval @ git+https://github.com/bzantium/koifeval.git@7a27889c0cf285e60b1a7b4795211846dc328aaf
+# CVE-2026-81726 is fixed upstream but has no release yet. Pin the first
+# immutable upstream merge containing all model-artifact pathsec fixes.
+nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad  # CVE-2026-81726
 langcodes
 langdetect
 language-data
diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills
index db97aeac0f..2927b95503 100644
--- a/dockerfiles/Dockerfile.nemo-skills
+++ b/dockerfiles/Dockerfile.nemo-skills
@@ -56,6 +56,7 @@ FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer
 # using ubuntu instead of debian for easier apptainer installation on arm64
 FROM ubuntu:22.04 AS runtime-base
 ARG WANDB_CORE_COMMIT
+ARG NLTK_SECURITY_COMMIT=574270e2ad368c8816976e584da56ddfb3fefbad
 
 # Install Python and other dependencies
 RUN apt-get update && \
@@ -119,7 +120,8 @@ RUN cd ${IFBENCH_DIR} && pip install -r requirements.txt
 COPY dockerfiles/ifbench.patch /opt/benchmarks/IFBench/ifbench.patch
 RUN cd /opt/benchmarks/IFBench && git apply ifbench.patch
 
-RUN pip install langdetect absl-py immutabledict "nltk>=3.10.3" ipython && \
+RUN pip install langdetect absl-py immutabledict \
+        "nltk @ git+https://github.com/nltk/nltk.git@${NLTK_SECURITY_COMMIT}" ipython && \
     python -c "import nltk; from spacy.cli import download; nltk.download('punkt'); nltk.download('punkt_tab'); \
     nltk.download('stopwords'); nltk.download('averaged_perceptron_tagger_eng'); download('en_core_web_sm')"
 
@@ -146,6 +148,7 @@ RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \
 # created it. Replacement prevents metadata from older pre-resolution packages
 # from surviving alongside the resolved versions.
 FROM runtime-base
+ARG NLTK_SECURITY_COMMIT
 RUN rm -rf /usr/local
 COPY --from=dependency-installer /usr/local/ /usr/local/
 # Replace W&B's vulnerable release binary with the source-compatible patched core
@@ -167,10 +170,13 @@ RUN site_packages="$(python -c "import sysconfig; print(sysconfig.get_paths()['p
 
 # Guard the final resolved environment and Ray's private import path against the
 # High findings seen in the multi-architecture image scans.
-RUN python -c "from importlib.metadata import version as v; from packaging.version import Version as V; \
+RUN python -c "import json; from importlib.metadata import distribution as d, version as v; from packaging.version import Version as V; \
                assert V(v('aiohttp')) >= V('3.14.3'), v('aiohttp'); \
                assert V(v('msgpack')) >= V('1.2.1'), v('msgpack'); \
                assert V(v('nltk')) >= V('3.10.3'), v('nltk'); \
+               direct = json.loads(d('nltk').read_text('direct_url.json')); expected = '${NLTK_SECURITY_COMMIT}'; \
+               assert direct['vcs_info']['requested_revision'] == expected, direct; \
+               assert direct['vcs_info']['commit_id'] == expected, direct; \
                assert V(v('starlette')) >= V('1.3.1'), v('starlette'); \
                assert V(v('setuptools')) >= V('78.1.1'), v('setuptools'); \
                print('aiohttp/msgpack/nltk/starlette/setuptools security floors OK')" && \
diff --git a/dockerfiles/Dockerfile.sandbox b/dockerfiles/Dockerfile.sandbox
index f256d27364..2b9bb8bd8a 100644
--- a/dockerfiles/Dockerfile.sandbox
+++ b/dockerfiles/Dockerfile.sandbox
@@ -18,6 +18,7 @@
 # To regenerate after changing code_execution.txt or stem.txt:
 #     uv pip compile requirements/code_execution.txt requirements/stem.txt \
 #         --extra-index-url https://download.pytorch.org/whl/cpu \
+#         --index-strategy unsafe-best-match \
 #         --universal -p 3.10 -o requirements/sandbox.lock
 # =============================================================================
 
diff --git a/pyproject.toml b/pyproject.toml
index 260a8a3505..9557751ebc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -90,7 +90,9 @@ override-dependencies = [
     # environment above the first fixed releases.
     "aiohttp>=3.14.3",
     "msgpack>=1.2.1",
-    "nltk>=3.10.3",
+    # CVE-2026-81726 has no fixed release. This is the first exact upstream
+    # merge containing the complete TransitionParser, MaxEnt, and tagger fixes.
+    "nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad",
     "setuptools>=78.1.1",
 ]
 
diff --git a/recipes/data-integrity/README.md b/recipes/data-integrity/README.md
index e1b125b437..135a496603 100644
--- a/recipes/data-integrity/README.md
+++ b/recipes/data-integrity/README.md
@@ -85,7 +85,8 @@ scripts/
 
 1. **Basic Requirements**
 ```bash
-pip install pandas numpy matplotlib seaborn scikit-learn nltk spacy \
+pip install pandas numpy matplotlib seaborn scikit-learn \
+    "nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad" spacy \
     sentence-transformers umap-learn plotly textstat textblob rouge \
     datasets tqdm openai
 ```
diff --git a/recipes/data-integrity/model_comparison/requirements.txt b/recipes/data-integrity/model_comparison/requirements.txt
index c4318c367d..df1936bedb 100644
--- a/recipes/data-integrity/model_comparison/requirements.txt
+++ b/recipes/data-integrity/model_comparison/requirements.txt
@@ -4,7 +4,7 @@ matplotlib>=3.4.0
 nbformat>=4.2.0
 
 # NLP and text processing
-nltk>=3.6.0
+nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad  # CVE-2026-81726 unreleased upstream fix
 numpy>=1.21.0
 
 # Additional dependencies that may be needed
diff --git a/requirements/sandbox.lock b/requirements/sandbox.lock
index aa18f968c4..a30a8e93b1 100644
--- a/requirements/sandbox.lock
+++ b/requirements/sandbox.lock
@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile requirements/code_execution.txt requirements/stem.txt --universal -p 3.10 -o requirements/sandbox.lock
+#    uv pip compile requirements/code_execution.txt requirements/stem.txt --index-strategy unsafe-best-match --universal -p 3.10 -o requirements/sandbox.lock
 absl-py==2.4.0
     # via
     #   array-record
@@ -13,8 +13,9 @@ absl-py==2.4.0
     #   tensorflow-datasets
 aiohappyeyeballs==2.6.1
     # via aiohttp
-aiohttp==3.13.5
+aiohttp==3.14.4
     # via
+    #   --override (workspace)
     #   datasets
     #   fsspec
     #   instructor
@@ -41,8 +42,9 @@ antlr4-python3-runtime==4.13.2
     # via
     #   lie
     #   liegentools
-anyio==4.13.0
+anyio==4.14.2
     # via
+    #   --override (workspace)
     #   httpx
     #   jupyter-server
     #   openai
@@ -392,7 +394,9 @@ deepdiff==9.0.0
     #   pandapower
     #   trx-python
 defusedxml==0.7.1
-    # via nbconvert
+    # via
+    #   nbconvert
+    #   nltk
 depmap==0.4.11
     # via -r requirements/stem.txt
 diff==2023.12.6
@@ -647,11 +651,13 @@ httpsproxy-urllib2==1.0
     # via gutenbergpy
 httpx==0.28.1
     # via
+    #   --override (workspace)
     #   biothings-client
     #   googletrans
     #   jupyterlab
     #   langsmith
     #   openai
+    #   rdflib
     #   weasel
     #   wikipedia-api
 httpx-sse==0.4.3
@@ -943,7 +949,7 @@ logistro==2.0.1
     #   kaleido
 luigi==3.8.0
     # via law
-lxml==6.0.4
+lxml==6.1.3
     # via
     #   -r requirements/stem.txt
     #   bioservices
@@ -1043,8 +1049,10 @@ mpmath==1.3.0
     # via sympy
 mpytools==0.0.28
     # via molparse
-msgpack==1.1.2
-    # via wordfreq
+msgpack==1.2.3
+    # via
+    #   --override (workspace)
+    #   wordfreq
 multidict==6.7.1
     # via
     #   aiohttp
@@ -1119,8 +1127,10 @@ nibabel==5.4.2
     #   trx-python
 ninja==1.13.0
     # via easyocr
-nltk==3.9.4
-    # via -r requirements/stem.txt
+nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad
+    # via
+    #   --override (workspace)
+    #   -r requirements/stem.txt
 notebook==7.1.3
     # via pyodesys
 notebook-shim==0.2.4
@@ -1947,8 +1957,9 @@ sentencepiece==0.2.1
     # via sanskrit-parser
 service-identity==24.2.0
     # via twisted
-setuptools==70.2.0
+setuptools==84.0.0
     # via
+    #   --override (workspace)
     #   gutenbergpy
     #   osqp
     #   pbr
@@ -2314,6 +2325,7 @@ types-pytz==2026.1.1.20260408
     # via pandas-stubs
 typing-extensions==4.15.0
     # via
+    #   aiohttp
     #   aiosignal
     #   anyio
     #   async-lru
@@ -2388,8 +2400,9 @@ uritemplate==4.2.0
     #   pycldf
 url-normalize==2.2.1
     # via requests-cache
-urllib3==1.26.13
+urllib3==2.8.0
     # via
+    #   --override (workspace)
     #   botocore
     #   requests
     #   requests-cache
@@ -2471,7 +2484,7 @@ xxhash==3.6.0
     # via
     #   datasets
     #   langsmith
-yarl==1.23.0
+yarl==1.25.1
     # via aiohttp
 yfinance==0.2.25
     # via -r requirements/stem.txt
diff --git a/requirements/stem.txt b/requirements/stem.txt
index 39a1a2d2f7..661681fdc1 100644
--- a/requirements/stem.txt
+++ b/requirements/stem.txt
@@ -92,7 +92,7 @@ mygene
 myvariant
 networkx
 nibabel
-nltk>=3.10.3  # fixes CVE-2026-79675, CVE-2026-71513, CVE-2026-72818, CVE-2026-78680
+nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad  # CVE-2026-81726 unreleased upstream fix
 nuclear
 num2words
 numba
diff --git a/security/README.md b/security/README.md
new file mode 100644
index 0000000000..dd41ab9090
--- /dev/null
+++ b/security/README.md
@@ -0,0 +1,22 @@
+# Security scan assertions
+
+`nltk-cve-2026-81726.openvex.json` is a narrow `fixed` statement for the
+repository's exact, audited NLTK source commit. NLTK has not published a release
+for CVE-2026-81726, so the patched checkout still reports version `3.10.3` and a
+version-only scanner cannot distinguish it from the vulnerable PyPI artifact.
+
+Run the dependency/provenance regression tests before applying the statement,
+and keep an unfiltered report for audit:
+
+```bash
+pytest -q tests/test_requirements_versions.py tests/test_dependency_functional.py
+trivy fs --scanners vuln --include-dev-deps \
+  --format json --output trivy-repository-raw.json --exit-code 0 .
+trivy fs --scanners vuln --include-dev-deps --skip-db-update \
+  --vex security/nltk-cve-2026-81726.openvex.json \
+  --format json --output trivy-repository.json --exit-code 0 .
+```
+
+Do not apply this VEX to an arbitrary NLTK `3.10.3` installation. Remove the
+Git source pin and this statement together when an official NLTK release newer
+than `3.10.3` is adopted.
diff --git a/security/nltk-cve-2026-81726.openvex.json b/security/nltk-cve-2026-81726.openvex.json
new file mode 100644
index 0000000000..03882e499e
--- /dev/null
+++ b/security/nltk-cve-2026-81726.openvex.json
@@ -0,0 +1,26 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://github.com/NVIDIA-NeMo/Skills/security/vex/nltk-cve-2026-81726/1",
+  "author": "NVIDIA NeMo-Skills maintainers",
+  "role": "Document Creator",
+  "timestamp": "2026-10-06T00:00:00-04:00",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {
+        "@id": "https://www.cve.org/CVERecord?id=CVE-2026-81726",
+        "name": "CVE-2026-81726",
+        "aliases": [
+          "GHSA-8mgp-746c-j5xp"
+        ]
+      },
+      "products": [
+        {
+          "@id": "pkg:pypi/nltk@3.10.3"
+        }
+      ],
+      "status": "fixed",
+      "status_notes": "NeMo-Skills installs NLTK from immutable upstream commit 574270e2ad368c8816976e584da56ddfb3fefbad, the first upstream merge containing the complete model-artifact path-containment fixes. Requirement, image-build provenance, and affected-API regression tests verify this assertion. Remove this statement when an official NLTK release newer than 3.10.3 is adopted."
+    }
+  ]
+}
diff --git a/tests/test_dependency_functional.py b/tests/test_dependency_functional.py
index 9e3d935302..f4a01527df 100644
--- a/tests/test_dependency_functional.py
+++ b/tests/test_dependency_functional.py
@@ -34,7 +34,7 @@
                                so guarded by importorskip)
   * aiohttp          >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
-  * nltk             >=3.10.3 (fixes CVE-2026-79675 and related High findings)
+  * nltk             pinned to upstream commit 574270e (fixes CVE-2026-81726)
   * starlette        >=1.3.1  (fixes CVE-2026-48818 and CVE-2026-54283)
   * setuptools       >=78.1.1 (fixes CVE-2025-47273)
 
@@ -44,12 +44,15 @@
 
 import asyncio
 import json
-from importlib.metadata import version
+import pickle
+from importlib.metadata import distribution, version
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
+NLTK_SECURITY_COMMIT = "574270e2ad368c8816976e584da56ddfb3fefbad"
+
 # ---------------------------------------------------------------------------
 # GitPython >=3.1.58 and datamodel-code-generator >=0.64.0
 # ---------------------------------------------------------------------------
@@ -83,6 +86,97 @@ def test_python_security_floors():
     assert Version(version("setuptools")) >= Version("78.1.1")
 
 
+def test_nltk_security_patch_provenance():
+    """Version 3.10.3 alone is vulnerable; prove this is the patched Git tree."""
+    direct_url_text = distribution("nltk").read_text("direct_url.json")
+    assert direct_url_text, "NLTK must be installed from the audited upstream commit"
+    direct_url = json.loads(direct_url_text)
+    assert direct_url["vcs_info"]["requested_revision"] == NLTK_SECURITY_COMMIT
+    assert direct_url["vcs_info"]["commit_id"] == NLTK_SECURITY_COMMIT
+
+
+def test_nltk_model_artifact_paths_stay_inside_pathsec(tmp_path, monkeypatch):
+    """Drive every CVE-2026-81726 model I/O sink against an outside path."""
+    import nltk.data
+    import numpy as np
+    from nltk import pathsec
+    from nltk.classify.maxent import save_maxent_params
+    from nltk.parse import DependencyGraph, transitionparser
+    from nltk.parse.transitionparser import TransitionParser
+    from nltk.tag.perceptron import AveragedPerceptron, PerceptronTagger
+
+    allowed = tmp_path / "allowed"
+    outside = tmp_path / "outside"
+    allowed.mkdir()
+    outside.mkdir()
+    monkeypatch.setattr(nltk.data, "path", [str(allowed)])
+    monkeypatch.setattr(pathsec, "ENFORCE", True)
+    monkeypatch.setattr(pathsec, "_get_allowed_roots", lambda: (str(allowed.resolve()),))
+
+    weights_path = outside / "weights.json"
+    with pytest.raises(PermissionError):
+        AveragedPerceptron({"feature": {"NN": 1.0}}).save(weights_path)
+    assert not weights_path.exists()
+
+    weights_path.write_text("{}", encoding="utf-8")
+    with pytest.raises(PermissionError):
+        AveragedPerceptron().load(weights_path)
+    assert weights_path.read_text(encoding="utf-8") == "{}"
+
+    tagger_path = outside / "tagger"
+    with pytest.raises(PermissionError):
+        PerceptronTagger(load=False).save_to_json(lang="eng", loc=tagger_path)
+    assert not tagger_path.exists()
+
+    maxent_path = outside / "maxent"
+    with pytest.raises(PermissionError):
+        save_maxent_params(np.array([1.0]), {}, [], {}, tab_dir=maxent_path)
+    assert not maxent_path.exists()
+
+    parser_model = outside / "parser.pickle"
+    parser_model.write_bytes(pickle.dumps({}))
+    with pytest.raises(PermissionError):
+        TransitionParser("arc-standard").parse([], str(parser_model))
+
+    class _Array:
+        def astype(self, *args, **kwargs):
+            return self
+
+    class _Features(_Array):
+        def __init__(self):
+            self.indices = _Array()
+            self.indptr = _Array()
+
+    class _Model:
+        def fit(self, *args, **kwargs):
+            return self
+
+    class _SVM:
+        @staticmethod
+        def SVC(*args, **kwargs):
+            return _Model()
+
+    graph = DependencyGraph("Economic\tJJ\t2\tATT\nnews\tNN\t3\tSBJ\nhas\tVBD\t0\tROOT\n")
+    trained_model = outside / "trained.pickle"
+    monkeypatch.setattr(
+        transitionparser,
+        "load_svmlight_file",
+        lambda _name: (_Features(), None),
+        raising=False,
+    )
+    monkeypatch.setattr(transitionparser, "svm", _SVM(), raising=False)
+    with pytest.raises(PermissionError):
+        TransitionParser("arc-standard").train([graph], str(trained_model), verbose=False)
+    assert not trained_model.exists()
+
+
+def test_nltk_tokenizers_still_work_after_security_patch():
+    from nltk.tokenize import PunktSentenceTokenizer, wordpunct_tokenize
+
+    assert PunktSentenceTokenizer().tokenize("One. Two.") == ["One.", "Two."]
+    assert wordpunct_tokenize("alpha, beta") == ["alpha", ",", "beta"]
+
+
 def test_fastapi_constructs_with_fixed_starlette():
     from fastapi import FastAPI
 
diff --git a/tests/test_requirements_versions.py b/tests/test_requirements_versions.py
index c657ca1bcc..51b4a4a327 100644
--- a/tests/test_requirements_versions.py
+++ b/tests/test_requirements_versions.py
@@ -23,7 +23,7 @@
   * lxml             -> >=6.1.0  (fixes GHSA-vfmq-68hx-4jfw)
   * aiohttp          -> >=3.14.3 (fixes CVE-2026-69244)
   * msgpack          -> >=1.2.1  (fixes GHSA-6v7p-g79w-8964)
-  * nltk             -> >=3.10.3 (fixes CVE-2026-79675 and related High findings)
+  * nltk             -> immutable upstream security commit for CVE-2026-81726
   * gradio           -> >=6.16.0 (fixes CVE-2026-49119)
   * starlette        -> >=1.3.1  (fixes CVE-2026-48818 and CVE-2026-54283)
   * setuptools       -> >=78.1.1 (fixes CVE-2025-47273)
@@ -40,6 +40,7 @@
 manual dependency resolve.
 """
 
+import json
 import re
 from pathlib import Path
 
@@ -52,10 +53,19 @@
 CORE_REQUIREMENTS = REPO_ROOT / "core" / "requirements.txt"
 PIPELINE_REQUIREMENTS = REPO_ROOT / "requirements" / "pipeline.txt"
 STEM_REQUIREMENTS = REPO_ROOT / "requirements" / "stem.txt"
+SANDBOX_LOCK = REPO_ROOT / "requirements" / "sandbox.lock"
+MODEL_COMPARISON_REQUIREMENTS = REPO_ROOT / "recipes" / "data-integrity" / "model_comparison" / "requirements.txt"
 PYPROJECT_TOML = REPO_ROOT / "pyproject.toml"
+NLTK_VEX = REPO_ROOT / "security" / "nltk-cve-2026-81726.openvex.json"
 BFCL_MODULE = REPO_ROOT / "nemo_skills" / "inference" / "eval" / "bfcl.py"
 NEMO_SKILLS_DOCKERFILE = REPO_ROOT / "dockerfiles" / "Dockerfile.nemo-skills"
-BUILD_PYPROJECTS = [PYPROJECT_TOML, REPO_ROOT / "core" / "pyproject.toml", REPO_ROOT / "tools" / "pyproject.toml"]
+BUILD_PYPROJECTS = [
+    PYPROJECT_TOML,
+    REPO_ROOT / "core" / "pyproject.toml",
+    REPO_ROOT / "tools" / "pyproject.toml",
+]
+NLTK_SECURITY_COMMIT = "574270e2ad368c8816976e584da56ddfb3fefbad"
+NLTK_SECURITY_URL = f"git+https://github.com/nltk/nltk.git@{NLTK_SECURITY_COMMIT}"
 
 
 def _load_toml(path: Path) -> dict:
@@ -96,6 +106,12 @@ def _find_requirement(path: Path, package_name: str) -> tuple[Requirement, str]:
     raise AssertionError(f"Could not find requirement '{package_name}' in {path}")
 
 
+def _assert_nltk_security_commit(req: Requirement) -> None:
+    assert req.name.lower() == "nltk"
+    assert not req.specifier, f"NLTK must use the exact patched source commit, not {req.specifier}"
+    assert req.url == NLTK_SECURITY_URL
+
+
 class TestCoreRequirements:
     """core/requirements.txt security floors and pins."""
 
@@ -138,6 +154,11 @@ def test_gradio_floor_fixes_cve_2026_49119(self):
         assert Version(specs[">="]) >= Version("6.16.0")
         assert "CVE-2026-49119" in comment
 
+    def test_nltk_uses_unreleased_upstream_cve_fix(self):
+        req, comment = _find_requirement(CORE_REQUIREMENTS, "nltk")
+        _assert_nltk_security_commit(req)
+        assert "CVE-2026-81726" in comment
+
     def test_bfcl_does_not_reinstall_vulnerable_datamodel_code_generator(self):
         content = BFCL_MODULE.read_text()
         assert '"datamodel-code-generator==0.64.0"' in content
@@ -209,6 +230,13 @@ def test_final_image_asserts_python_security_floors(self, dockerfile):
         assert "V(v('starlette')) >= V('1.3.1')" in dockerfile
         assert "V(v('setuptools')) >= V('78.1.1')" in dockerfile
 
+    def test_nltk_patch_source_and_provenance_are_verified(self, dockerfile):
+        assert f"NLTK_SECURITY_COMMIT={NLTK_SECURITY_COMMIT}" in dockerfile
+        assert "github.com/nltk/nltk.git@${NLTK_SECURITY_COMMIT}" in dockerfile
+        assert "d('nltk').read_text('direct_url.json')" in dockerfile
+        assert "direct['vcs_info']['requested_revision'] == expected" in dockerfile
+        assert "direct['vcs_info']['commit_id'] == expected" in dockerfile
+
     def test_ray_private_aiohttp_and_uv_wheel_are_remediated(self, dockerfile):
         assert "ray/_private/runtime_env/agent/thirdparty_files" in dockerfile
         assert "FROM ghcr.io/astral-sh/uv:0.12.9 AS uv-installer" in dockerfile
@@ -283,12 +311,14 @@ def test_lxml_is_not_unbounded_or_unpinned(self):
             if code_part == "lxml":
                 pytest.fail(f"lxml requirement has no version floor: {raw_line!r}")
 
-    def test_nltk_floor_fixes_current_critical_and_high_findings(self):
+    def test_nltk_uses_unreleased_upstream_cve_fix(self):
         req, comment = _find_requirement(STEM_REQUIREMENTS, "nltk")
-        specs = {spec.operator: spec.version for spec in req.specifier}
-        assert ">=" in specs, f"expected a floor (>=) specifier for nltk, got {req.specifier}"
-        assert Version(specs[">="]) >= Version("3.10.3")
-        assert "CVE-2026-79675" in comment
+        _assert_nltk_security_commit(req)
+        assert "CVE-2026-81726" in comment
+
+    def test_sandbox_lock_uses_same_nltk_security_commit(self):
+        req, _ = _find_requirement(SANDBOX_LOCK, "nltk")
+        _assert_nltk_security_commit(req)
 
 
 class TestPyprojectUvDependencyPolicy:
@@ -335,7 +365,6 @@ def test_urllib3_override_present(self, uv_overrides):
             ("aiohttp", "3.14.3"),
             ("anyio", "4.14.2"),
             ("msgpack", "1.2.1"),
-            ("nltk", "3.10.3"),
             ("prometheus-fastapi-instrumentator", "8.1.0"),
             ("setuptools", "78.1.1"),
         ],
@@ -346,6 +375,10 @@ def test_container_security_override_present(self, uv_overrides, package, minimu
         assert ">=" in specs
         assert Version(specs[">="]) >= Version(minimum)
 
+    def test_nltk_override_uses_unreleased_upstream_cve_fix(self, uv_overrides):
+        assert "nltk" in uv_overrides
+        _assert_nltk_security_commit(uv_overrides["nltk"])
+
     @pytest.mark.parametrize(
         ("package", "minimum"),
         [
@@ -366,6 +399,25 @@ def test_dependencies_still_sourced_from_core_and_pipeline_requirements(self):
         assert dynamic_deps["file"] == ["core/requirements.txt", "requirements/pipeline.txt"]
 
 
+def test_nltk_vex_is_narrow_and_tied_to_the_audited_commit():
+    document = json.loads(NLTK_VEX.read_text())
+    assert document["@context"] == "https://openvex.dev/ns/v0.2.0"
+    assert document["version"] == 1
+    assert len(document["statements"]) == 1
+    statement = document["statements"][0]
+    assert statement["vulnerability"]["name"] == "CVE-2026-81726"
+    assert statement["products"] == [{"@id": "pkg:pypi/nltk@3.10.3"}]
+    assert statement["status"] == "fixed"
+    assert NLTK_SECURITY_COMMIT in statement["status_notes"]
+
+
+def test_data_integrity_recipe_uses_same_nltk_security_commit():
+    req, _ = _find_requirement(MODEL_COMPARISON_REQUIREMENTS, "nltk")
+    _assert_nltk_security_commit(req)
+    readme = (REPO_ROOT / "recipes" / "data-integrity" / "README.md").read_text()
+    assert NLTK_SECURITY_URL in readme
+
+
 class TestPyprojectCommentUpdated:
     """The comment above override-dependencies referenced an exact litellm pin
     that has since moved; make sure it was updated rather than left stale."""

From 73dda2010a4664e6d5e9a9c7f68aaa66729b197a Mon Sep 17 00:00:00 2001
From: Nick Gupta <nicgupta@nvidia.com>
Date: Tue, 6 Oct 2026 13:32:49 -0400
Subject: [PATCH 30/30] fix(ci): keep requirements sorted

Signed-off-by: Nick Gupta <nicgupta@nvidia.com>
---
 core/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/requirements.txt b/core/requirements.txt
index b754ce9bf7..c59cfb47d9 100644
--- a/core/requirements.txt
+++ b/core/requirements.txt
@@ -24,9 +24,6 @@ hydra-core
 ipython
 iso639-lang
 koifeval @ git+https://github.com/bzantium/koifeval.git@7a27889c0cf285e60b1a7b4795211846dc328aaf
-# CVE-2026-81726 is fixed upstream but has no release yet. Pin the first
-# immutable upstream merge containing all model-artifact pathsec fixes.
-nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad  # CVE-2026-81726
 langcodes
 langdetect
 language-data
@@ -37,6 +34,9 @@ math-verify[antlr4_9_3]
 # mcp 2.0 removed the streamablehttp_client alias that nemo_skills/mcp/clients.py imports.
 # Unpin once those call sites move to streamable_http_client and the 2.0 API is verified.
 mcp<2.0
+# CVE-2026-81726 is fixed upstream but has no release yet. Pin the first
+# immutable upstream merge containing all model-artifact pathsec fixes.
+nltk @ git+https://github.com/nltk/nltk.git@574270e2ad368c8816976e584da56ddfb3fefbad  # CVE-2026-81726
 numpy
 openai
 openpyxl>=3.1.0