diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b04042bfac5..ae5e97e0afd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1923,6 +1923,7 @@ jobs: E2E_JOB: "1" E2E_TARGET_ID: "gpu-e2e" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/gpu-e2e + E2E_LLAMA_CPP_DEDICATED_LANE: "1" NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -1977,6 +1978,71 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + llama-cpp-generic-gpu: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',llama-cpp-generic-gpu,') || contains(format(',{0},', inputs.targets), ',llama-cpp-generic-gpu,') }} + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 120 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "llama-cpp-generic-gpu" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-generic-gpu + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha || github.sha }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_PROVIDER: "install-llama-cpp" + NEMOCLAW_LLAMACPP_RECIPE: "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1" + NEMOCLAW_SANDBOX_NAME: "e2e-llamacpp-gpu" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + with: + build-cli: "false" + + - name: Restore exact-commit CLI artifact + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3 + with: + provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run generic NVIDIA GPU llama.cpp live test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/llama-cpp-generic-gpu.test.ts + + - name: Upload generic NVIDIA GPU llama.cpp artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + # Every main push runs this lane. Manual PR qualification also requires the # exact candidate activation contract. managed-image-multiarch-startup: @@ -7261,6 +7327,7 @@ jobs: inference-routing, cloud-inference, gpu-e2e, + llama-cpp-generic-gpu, managed-image-multiarch-startup, llama-cpp-dgx-spark-plan, llama-cpp-dgx-spark-qualification, diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index c6d136a6431..db0dc7bf9e2 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -36,6 +36,109 @@ jobs: - id: get-pr-info uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main + select-llama-cpp-generic-gpu: + needs: get-pr-info + runs-on: ubuntu-latest + outputs: + selected: ${{ steps.changed.outputs.selected }} + steps: + - id: changed + name: Select llama.cpp generic GPU E2E from PR files + env: + GH_TOKEN: ${{ github.token }} + PR_INFO: ${{ needs.get-pr-info.outputs.pr-info }} + shell: bash + run: | + set -euo pipefail + pr_number="$(jq -er '.number | select(type == "number" and . > 0)' <<<"$PR_INFO")" + head_sha="$(jq -er '.head.sha | select(test("^[a-f0-9]{40}$"))' <<<"$PR_INFO")" + [[ "$head_sha" == "$GITHUB_SHA" ]] || { + echo "::error::Copied PR branch SHA does not match the current PR head" >&2 + exit 1 + } + if gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/pulls/$pr_number/files?per_page=100" \ + | jq -e ' + flatten + | any( + .filename == ".github/workflows/e2e.yaml" + or .filename == ".github/workflows/pr-self-hosted.yaml" + or .filename == "test/e2e/live/llama-cpp-generic-gpu.test.ts" + or .filename == "test/e2e/live/gpu-e2e-helpers.ts" + or .filename == "test/e2e/mock-parity.json" + or .filename == "tools/e2e/cli-artifact-workflow-boundary.mts" + or .filename == "tools/e2e/workflow-boundary.mts" + or (.filename | startswith("managed-inference/presets/llama-cpp.")) + or (.filename | startswith("managed-inference/recipes/llama-cpp.")) + or (.filename | startswith("src/lib/inference/llama-cpp/")) + or (.filename | startswith("src/lib/onboard/runtime-provider/docker-llama-cpp")) + ) + ' >/dev/null; then + selected=true + else + selected=false + fi + printf 'selected=%s\n' "$selected" >>"$GITHUB_OUTPUT" + + llama-cpp-generic-gpu: + name: llama.cpp on generic NVIDIA GPU + needs: + - get-pr-info + - select-llama-cpp-generic-gpu + if: ${{ needs.select-llama-cpp-generic-gpu.outputs.selected == 'true' }} + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 120 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-generic-gpu + E2E_JOB: "1" + E2E_TARGET_ID: llama-cpp-generic-gpu + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_E2E_EXPECTED_SHA: ${{ fromJSON(needs.get-pr-info.outputs.pr-info).head.sha }} + NEMOCLAW_E2E_SHARD: default + NEMOCLAW_LLAMACPP_RECIPE: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1 + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_PROVIDER: install-llama-cpp + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_SANDBOX_NAME: e2e-llamacpp-gpu + OPENSHELL_GATEWAY: nemoclaw + steps: + - name: Checkout exact PR head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ fromJSON(needs.get-pr-info.outputs.pr-info).head.sha }} + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Bind E2E correlation identity + shell: bash + run: | + set -euo pipefail + correlation_id="$(node --input-type=module -e \ + 'import { randomUUID } from "node:crypto"; console.log(randomUUID())')" + [[ "$correlation_id" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] + printf 'NEMOCLAW_E2E_CORRELATION_ID=%s\n' "$correlation_id" >>"$GITHUB_ENV" + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run llama.cpp generic NVIDIA GPU live test + shell: bash + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + OPENSHELL_BIN="$(command -v openshell)" + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/llama-cpp-generic-gpu.test.ts + + - name: Upload llama.cpp generic NVIDIA GPU artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + build-sandbox-images: runs-on: linux-amd64-cpu4 timeout-minutes: 45 diff --git a/managed-inference/presets/llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b.yaml b/managed-inference/presets/llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b.yaml new file mode 100644 index 00000000000..9ad7495640b --- /dev/null +++ b/managed-inference/presets/llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b.yaml @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset + +metadata: + id: llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b + displayName: NVIDIA Nemotron 3 Nano 30B-A3B on one Linux x86_64 NVIDIA GPU + supportState: experimental + +spec: + selection: explicit-only + priority: 440 + + requirements: + all: + - readiness: + scope: everyNode + kind: capability + id: host.platform.supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.daemon_reachable + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.runtime_supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.storage_compatible + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.nvidia_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.container_toolkit_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.cdi_healthy + state: present + - readiness: + scope: everyNode + kind: observation + id: host.os.platform + comparison: + operator: equals + value: linux + - readiness: + scope: everyNode + kind: observation + id: host.os.architecture + comparison: + operator: equals + value: x64 + - readiness: + scope: everyNode + kind: observation + id: host.docker.runtime + comparison: + operator: equals + value: docker + - readiness: + scope: everyNode + kind: observation + id: host.gpu.count + comparison: + operator: at-least + value: 1 + - readiness: + scope: everyNode + kind: observation + id: host.gpu.driver_version + comparison: + operator: version-at-least + value: 580.65.06 + + plan: + backend: install-llama-cpp + recipeRef: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1 diff --git a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml index d61e9d20522..5527231d794 100644 --- a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml +++ b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml @@ -31,7 +31,7 @@ spec: license: NVIDIA-Open-Model-License acquisition: ref: hugging-face-exact-file/v1 - downloaderImage: nvcr.io/nvidia/vllm@sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2 + downloaderImage: nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03 authentication: mode: optional environment: HF_TOKEN @@ -92,7 +92,7 @@ spec: contractRef: llama-cpp.server-readiness/v1 timeoutSeconds: 1800 expectedModel: nvidia-nemotron-3-nano-30b-a3b - probeImage: nvcr.io/nvidia/vllm@sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2 + probeImage: nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03 probes: models: true health: true diff --git a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts index ad690b08a6d..fbd6213da9e 100644 --- a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts +++ b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts @@ -37,7 +37,7 @@ export const LLAMA_CPP_DGX_SPARK_CUDA_DEVELOPMENT_BASE = export const LLAMA_CPP_DGX_SPARK_CUDA_RUNTIME_BASE = "docker.io/nvidia/cuda@sha256:789e629e49401647e22b7054ae9c6c4f6427dba68010ba428deb4cc6b063676e" as const; export const LLAMA_CPP_DGX_SPARK_TOOL_IMAGE = - "nvcr.io/nvidia/vllm@sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2" as const; + "nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03" as const; export const LLAMA_CPP_DGX_SPARK_MINIMUM_DRIVER_VERSION = "580.65.06" as const; export const LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES = [ "health", diff --git a/src/lib/inference/llama-cpp/host-local-runtime.test.ts b/src/lib/inference/llama-cpp/host-local-runtime.test.ts index 4a17143f6bc..a8bd81d4da6 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.test.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.test.ts @@ -135,7 +135,7 @@ describe("llama.cpp host-local runtime materializer", () => { `type=bind,source=${runtime.model.hostPath},target=/models/${input.model.file.path},readonly`, `type=bind,source=${runtime.apiKeyHostPath},target=/run/secrets/llama-cpp-api-key,readonly`, ]); - expect(valuesAfter(argv, "--publish")).toEqual(["127.0.0.1::8081"]); + expect(valuesAfter(argv, "--publish")).toEqual([]); expect(valuesAfter(argv, "--gpus")).toEqual(["driver=nvidia,count=1"]); expect(valuesAfter(argv, "--gpu-layers")).toEqual(["all"]); expect(valuesAfter(argv, "--ctx-size")).toEqual([String(input.serve.contextSize)]); @@ -160,13 +160,13 @@ describe("llama.cpp host-local runtime materializer", () => { expect(argv.join("\n")).not.toContain("huggingface.co"); }); - it("publishes the fixed loopback host port when the bindings pin one", () => { + it("leaves fixed host-port bridging to the Docker lifecycle provider", () => { const argv = buildLlamaCppHostLocalDockerArgv(contract(), { ...bindings(), hostPort: LLAMA_CPP_PORT, }); - expect(valuesAfter(argv, "--publish")).toEqual([`127.0.0.1:${String(LLAMA_CPP_PORT)}:8081`]); + expect(valuesAfter(argv, "--publish")).toEqual([]); }); it("takes launch settings from the declared contract instead of code defaults (#8144)", () => { @@ -206,8 +206,6 @@ describe("llama.cpp host-local runtime materializer", () => { "unless-stopped", "--user", "1001:1001", - "--publish", - "127.0.0.1::8081", "--read-only", "--cap-drop", "ALL", diff --git a/src/lib/inference/llama-cpp/host-local-runtime.ts b/src/lib/inference/llama-cpp/host-local-runtime.ts index 8f6c97e2f37..af93acca8b3 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.ts @@ -82,7 +82,7 @@ export interface LlamaCppHostLocalRuntimeBindings { readonly apiKeyHostPath: string; readonly containerName: string; readonly imageReference: string; - /** Fixed loopback host port for product installs; omitted only by isolated qualification. */ + /** Fixed host bridge port for product installs; omitted only by isolated qualification. */ readonly hostPort?: number; readonly model: VerifiedLocalModelArtifact; /** The caller must create this named Docker network with `--internal` before launch. */ @@ -286,8 +286,6 @@ export function buildLlamaCppHostLocalDockerArgv( contract.runtime.restartPolicy, "--user", runtimeIdentity, - "--publish", - `127.0.0.1:${bindings.hostPort === undefined ? "" : String(bindings.hostPort)}:${String(serve.port)}`, "--read-only", "--cap-drop", "ALL", diff --git a/src/lib/inference/llama-cpp/managed-installer.ts b/src/lib/inference/llama-cpp/managed-installer.ts index 6007cb52ea2..de3e90cb2bc 100644 --- a/src/lib/inference/llama-cpp/managed-installer.ts +++ b/src/lib/inference/llama-cpp/managed-installer.ts @@ -350,15 +350,25 @@ export function resolveManagedLlamaCppOwnerSelection( throw new Error("Managed llama.cpp catalog authority changed; rerun onboarding."); } const recipe = catalog.recipes.find(({ metadata }) => metadata.id === owner.recipeId); - const preset = catalog.presets.find( + const candidatePresets = catalog.presets.filter( ({ spec }) => spec.selection === "explicit-only" && spec.plan.backend === "install-llama-cpp" && spec.plan.recipeRef === owner.recipeId, ); - if (!recipe || !isLlamaCppServingRecipe(recipe) || !preset) { + if (!recipe || !isLlamaCppServingRecipe(recipe) || candidatePresets.length === 0) { throw new Error("Managed llama.cpp declarative authority is unavailable."); } + const matchingPresets = candidatePresets.filter(({ metadata }) => + catalog.sources.some( + ({ kind, id, digest }) => + kind === "ServingPreset" && id === metadata.id && digest === owner.presetDigest, + ), + ); + if (matchingPresets.length !== 1) { + throw new Error("Managed llama.cpp recipe authority changed; rerun onboarding."); + } + const preset = matchingPresets[0]!; const recipeDigest = catalog.sources.find( ({ kind, id }) => kind === "ServingRecipe" && id === recipe.metadata.id, )?.digest; diff --git a/src/lib/inference/llama-cpp/managed-selection.test.ts b/src/lib/inference/llama-cpp/managed-selection.test.ts index d96cd8b96b7..4dfdccea8db 100644 --- a/src/lib/inference/llama-cpp/managed-selection.test.ts +++ b/src/lib/inference/llama-cpp/managed-selection.test.ts @@ -10,6 +10,8 @@ import { LLAMA_CPP_RECIPE_ENV } from "./contract"; import { resolveManagedLlamaCppSelection } from "./managed-selection"; const RECIPE_ID = "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1"; +const GENERIC_PRESET_ID = "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b"; +const SPARK_PRESET_ID = "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b"; function readinessReport( preset: ManagedInferenceServingPreset, @@ -58,11 +60,9 @@ function readinessReport( } as SystemReadinessReport; } -function fixture() { +function fixture(presetId = SPARK_PRESET_ID) { const catalog = loadManagedInferenceCatalog(); - const preset = catalog.presets.find( - ({ spec }) => spec.plan.backend === "install-llama-cpp" && spec.plan.recipeRef === RECIPE_ID, - ); + const preset = catalog.presets.find(({ metadata }) => metadata.id === presetId); expect(preset, "Shipped managed llama.cpp preset is missing.").toBeDefined(); return { catalog, preset: preset!, report: readinessReport(preset!) }; } @@ -135,6 +135,90 @@ describe("managed llama.cpp selection", () => { }); }); + it("selects the generic Linux amd64 NVIDIA GPU preset from the same declarative recipe", () => { + const { catalog } = fixture(GENERIC_PRESET_ID); + const now = new Date(); + const report = createHostReadinessReport( + { + nemoclawVersion: "0.1.0", + sourceRevision: "a".repeat(40), + now: () => now, + }, + { + now: () => now, + architecture: "x64", + assess: () => ({ + platform: "linux", + isWsl: false, + runtime: "docker", + dockerInstalled: true, + dockerRunning: true, + dockerReachable: true, + nodeInstalled: true, + openshellInstalled: true, + dockerCgroupVersion: "v2", + dockerDefaultCgroupnsMode: "private", + dockerStorageDriver: "overlay2", + dockerUsesContainerdSnapshotter: false, + dockerCpus: 16, + dockerMemTotalBytes: 64 * 1024 ** 3, + isContainerRuntimeUnderProvisioned: false, + hasNestedOverlayConflict: false, + requiresHostCgroupnsFix: false, + isUnsupportedRuntime: false, + isHeadlessLikely: true, + hasNvidiaGpu: true, + dockerCdiSpecDirs: ["/etc/cdi"], + cdiNvidiaGpuSpecMissing: false, + cdiNvidiaGpuSpecStale: false, + cdiNvidiaGpuSpecNeedsRepair: false, + nvidiaContainerToolkitInstalled: true, + notes: [], + }), + collectPlatformIdentity: () => ({ + nvidiaPlatform: "linux", + productName: "NVIDIA RTX PRO 6000 Blackwell Server Edition", + }), + detectGpu: () => ({ count: 1 }), + detectHostGpuPlatform: () => "linux", + detectNvidiaDriverVersion: () => "580.65.06", + }, + ); + + const resolved = resolveManagedLlamaCppSelection( + { [LLAMA_CPP_RECIPE_ENV]: RECIPE_ID }, + catalog, + report, + ); + + expect(resolved).toMatchObject({ + kind: "selected", + selection: { + recipe: { metadata: { id: RECIPE_ID } }, + preset: { metadata: { id: GENERIC_PRESET_ID } }, + }, + }); + }); + + it("rejects a host that matches more than one hardware preset for one recipe", () => { + const { catalog, preset, report } = fixture(GENERIC_PRESET_ID); + const duplicate = { + ...preset, + metadata: { ...preset.metadata, id: `${GENERIC_PRESET_ID}.duplicate` }, + }; + + const resolved = resolveManagedLlamaCppSelection( + { [LLAMA_CPP_RECIPE_ENV]: RECIPE_ID }, + { ...catalog, presets: [...catalog.presets, duplicate] }, + report, + ); + + expect(resolved).toEqual({ + kind: "rejected", + reason: `Managed llama.cpp recipe ${RECIPE_ID} matches more than one explicit serving preset: ${GENERIC_PRESET_ID}, ${GENERIC_PRESET_ID}.duplicate.`, + }); + }); + it("selects an explicitly named shipped recipe", () => { const { catalog, report } = fixture(); diff --git a/src/lib/inference/llama-cpp/managed-selection.ts b/src/lib/inference/llama-cpp/managed-selection.ts index f8ebeb326b6..9b6e41daca9 100644 --- a/src/lib/inference/llama-cpp/managed-selection.ts +++ b/src/lib/inference/llama-cpp/managed-selection.ts @@ -23,17 +23,19 @@ export type ManagedLlamaCppSelectionResult = | { readonly kind: "selected"; readonly selection: ResolvedLlamaCppInferenceSelection } | { readonly kind: "rejected"; readonly reason: string }; -function uniquePresetForRecipe( +function explicitPresetsForRecipe( catalog: CompiledManagedInferenceCatalog, requestedRecipeId: string, -): string | null { - const matches = catalog.presets.filter( - (preset) => - preset.spec.selection === "explicit-only" && - preset.spec.plan.backend === "install-llama-cpp" && - preset.spec.plan.recipeRef === requestedRecipeId, - ); - return matches.length === 1 ? matches[0]!.metadata.id : null; +): readonly string[] { + return catalog.presets + .filter( + (preset) => + preset.spec.selection === "explicit-only" && + preset.spec.plan.backend === "install-llama-cpp" && + preset.spec.plan.recipeRef === requestedRecipeId, + ) + .map(({ metadata }) => metadata.id) + .sort((left, right) => left.localeCompare(right)); } function defaultRecipeId(catalog: CompiledManagedInferenceCatalog): string | null { @@ -63,11 +65,11 @@ export function resolveManagedLlamaCppSelection( reason: `${LLAMA_CPP_RECIPE_ENV} must name one supported declarative recipe.`, }; } - const presetId = uniquePresetForRecipe(catalog, recipeId); - if (!presetId) { + const presetIds = explicitPresetsForRecipe(catalog, recipeId); + if (presetIds.length === 0) { return { kind: "rejected", - reason: `Managed llama.cpp recipe ${recipeId} does not resolve one explicit serving preset.`, + reason: `Managed llama.cpp recipe ${recipeId} does not resolve an explicit serving preset.`, }; } if (String(env.NEMOCLAW_MODEL ?? "").trim()) { @@ -76,17 +78,38 @@ export function resolveManagedLlamaCppSelection( reason: `NEMOCLAW_MODEL cannot override the served model in ${LLAMA_CPP_RECIPE_ENV}.`, }; } - const resolution = resolveManagedInferenceServing( - { - readinessReports: [{ nodeId: os.hostname(), report }], - topologyQualifications: [], - intent: { provider: "install-llama-cpp", preset: presetId }, - }, - catalog, - ); - if (resolution.outcome !== "selected") { - return { kind: "rejected", reason: resolution.message }; + const resolutions = presetIds.map((presetId) => ({ + presetId, + resolution: resolveManagedInferenceServing( + { + readinessReports: [{ nodeId: os.hostname(), report }], + topologyQualifications: [], + intent: { provider: "install-llama-cpp", preset: presetId }, + }, + catalog, + ), + })); + const selected = resolutions.filter(({ resolution }) => resolution.outcome === "selected"); + if (selected.length !== 1) { + if (selected.length > 1) { + return { + kind: "rejected", + reason: `Managed llama.cpp recipe ${recipeId} matches more than one explicit serving preset: ${selected.map(({ presetId }) => presetId).join(", ")}.`, + }; + } + return { + kind: "rejected", + reason: `Managed llama.cpp recipe ${recipeId} does not match this host: ${resolutions + .map(({ presetId, resolution }) => + resolution.outcome === "selected" + ? `${presetId}: matched unexpectedly` + : `${presetId}: ${resolution.message}`, + ) + .join("; ")}`, + }; } + const resolution = selected[0]!.resolution; + if (resolution.outcome !== "selected") throw new Error("Selected resolution is unavailable."); if ( !isLlamaCppServingRecipe(resolution.recipe) || resolution.recipe.spec.execution.materializerRef !== LLAMA_CPP_HOST_LOCAL_MATERIALIZER_REF || diff --git a/src/lib/inference/llama-cpp/managed-status.test.ts b/src/lib/inference/llama-cpp/managed-status.test.ts index 3334f60c1fe..0bfc5d8a673 100644 --- a/src/lib/inference/llama-cpp/managed-status.test.ts +++ b/src/lib/inference/llama-cpp/managed-status.test.ts @@ -31,6 +31,7 @@ const RUNTIME_ID = "2".repeat(64); const SPEC_SHA256 = "3".repeat(64); const NETWORK_ID = "e".repeat(64); const RECIPE_ID = "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1"; +const GENERIC_PRESET_ID = "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b"; const IMAGE = `ghcr.io/nvidia/llama-cpp@sha256:${"4".repeat(64)}`; const temporaryDirectories: string[] = []; @@ -60,19 +61,23 @@ function engine( }; } -function reserveState(homeDir: string): void { +function reserveState(homeDir: string, presetId?: string, presetDigest?: string): void { const paths = managedLlamaCppStatePaths(homeDir); const catalog = loadManagedInferenceCatalog(); const preset = catalog.presets.find( - ({ spec }) => spec.plan.backend === "install-llama-cpp" && spec.plan.recipeRef === RECIPE_ID, + ({ metadata, spec }) => + spec.plan.backend === "install-llama-cpp" && + spec.plan.recipeRef === RECIPE_ID && + (presetId === undefined || metadata.id === presetId), )!; reserveManagedLlamaCppOwner(paths, { schemaVersion: 1, sandboxName: "spark-agent", catalogDigest: catalog.catalogDigest, - presetDigest: catalog.sources.find( - ({ kind, id }) => kind === "ServingPreset" && id === preset.metadata.id, - )!.digest, + presetDigest: + presetDigest ?? + catalog.sources.find(({ kind, id }) => kind === "ServingPreset" && id === preset.metadata.id)! + .digest, recipeDigest: catalog.sources.find( ({ kind, id }) => kind === "ServingRecipe" && id === RECIPE_ID, )!.digest, @@ -162,6 +167,34 @@ describe("managed llama.cpp status", () => { }); }); + it("revalidates the selected preset when multiple presets share one recipe (#8144)", () => { + const homeDir = temporaryHome(); + reserveState(homeDir, GENERIC_PRESET_ID); + + expect(inspectManagedLlamaCppStatus("spark-agent", { homeDir })).toEqual({ + recipeId: RECIPE_ID, + modelDigest: null, + imageReference: null, + endpoint: "https://inference.local/v1", + state: "preparing", + detail: "ownership is reserved; no finalized runtime receipt is published", + }); + }); + + it("rejects an unrecognized preset digest before Docker inspection (#8144)", () => { + const homeDir = temporaryHome(); + reserveState(homeDir, GENERIC_PRESET_ID, `sha256:${"f".repeat(64)}`); + const capture = vi.fn(); + + expect( + inspectManagedLlamaCppStatus("spark-agent", { homeDir, engine: engine(capture) }), + ).toMatchObject({ + state: "unknown", + detail: "Managed llama.cpp recipe authority changed; rerun onboarding.", + }); + expect(capture).not.toHaveBeenCalled(); + }); + it("reports the exact receipt-bound container as absent without further inspection", () => { const inspectExact = vi.fn(); const probe = vi.fn(); diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index d7930981de7..88e75d13e2f 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -36,11 +36,13 @@ const EXPECTED_MANAGED_RECIPE_IDS = [ ]; const EXPECTED_MANAGED_PRESET_IDS = [ "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b", + "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b", "local-model-profile.vllm.spark.v1", "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731", ]; const EXPECTED_MANAGED_SOURCE_IDS = [ "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b", + "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b", "local-model-profile.vllm.spark.v1", "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731", "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1", diff --git a/src/lib/inference/serving/catalog.ts b/src/lib/inference/serving/catalog.ts index a776488fa2f..2d659ca91a6 100644 --- a/src/lib/inference/serving/catalog.ts +++ b/src/lib/inference/serving/catalog.ts @@ -385,10 +385,11 @@ function llamaCppReadinessComparisonMatches( if (role !== "architecture" || actual.operator !== "equals") { return canonicalReadinessComparison(actual) === canonicalReadinessComparison(expected); } + const architecture = actual.value === "x64" ? "amd64" : actual.value; return ( - typeof actual.value === "string" && + typeof architecture === "string" && recipe.spec.runtime.platforms.includes( - `linux/${actual.value}` as LlamaCppServingRecipe["spec"]["runtime"]["platforms"][number], + `linux/${architecture}` as LlamaCppServingRecipe["spec"]["runtime"]["platforms"][number], ) ); } diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts index 3076523ec99..14832a519ca 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts @@ -6,7 +6,6 @@ import { createHash } from "node:crypto"; import { LLAMA_CPP_PORT } from "../../inference/llama-cpp/contract"; import type { LlamaCppHostLocalLaunchContract } from "../../inference/llama-cpp/host-local-runtime"; -export const HOST_PORT = String(LLAMA_CPP_PORT); export const MODEL_DIGEST = `sha256:${"a".repeat(64)}`; export const IMAGE = `ghcr.io/nvidia/nemoclaw/llama-cpp-server@sha256:${"c".repeat(64)}`; export const PROBE_IMAGE = `quay.io/curl/curl@sha256:${"d".repeat(64)}`; diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts index a644bf9ddfa..d0709f706c3 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test.ts @@ -12,14 +12,10 @@ import type { ContainerEngine } from "../../adapters/container-engine"; import { LLAMA_CPP_PORT } from "../../inference/llama-cpp/contract"; import type { LlamaCppGgufCachePlan } from "../../inference/llama-cpp/gguf-cache-plan"; import { buildLlamaCppHostLocalServerArgv } from "../../inference/llama-cpp/host-local-runtime"; -import { - createDockerLlamaCppManagedLifecycle, - type DockerLlamaCppManagedLifecycleOptions, -} from "./docker-llama-cpp-managed-lifecycle"; +import type { DockerLlamaCppManagedLifecycleOptions } from "./docker-llama-cpp-managed-lifecycle"; import { contract, digest, - HOST_PORT, IMAGE, invariant, MODEL_CONTENT, @@ -33,6 +29,10 @@ import { rawDigest, TRANSACTION_ID, } from "./docker-llama-cpp-managed-lifecycle.test-support"; +import { + createTestDockerLlamaCppManagedLifecycle as createLifecycle, + privateBridgeFixture, +} from "./docker-llama-cpp-private-bridge.test-support"; import type { HostLocalCreateJournalExecutionLease, HostLocalCreateJournalRecord, @@ -40,7 +40,6 @@ import type { } from "./host-local-create-journal"; import { type HostLocalInferenceReceiptWriter, - parseHostLocalInferenceReceipt, serializeHostLocalInferenceReceipt, } from "./host-local-inference"; import type { PersistedEngineAuthorityStore } from "./persisted-engine-authority"; @@ -307,10 +306,10 @@ interface DockerFixture { } function dockerFixture( - configuredHostPort = HOST_PORT, + configuredHostPort = "", publishedHostPort?: string, publishedHostIp = "127.0.0.1", - publishedBindingCount = 1, + publishedBindingCount = 0, ): DockerFixture { const effectivePublishedHostPort = publishedHostPort ?? (configuredHostPort || "49152"); let networkId = NETWORK_ID; @@ -347,7 +346,6 @@ function dockerFixture( command: string[]; } | undefined; - const inspection = () => [ { Id: RUNTIME_ID, @@ -361,9 +359,9 @@ function dockerFixture( HostConfig: { NetworkMode: "nemoclaw-llama-cpp-internal", RestartPolicy: { Name: "unless-stopped", MaximumRetryCount: 0 }, - PortBindings: { - "8081/tcp": [{ HostIp: "127.0.0.1", HostPort: configuredHostPort }], - }, + PortBindings: configuredHostPort + ? { "8081/tcp": [{ HostIp: "127.0.0.1", HostPort: configuredHostPort }] } + : {}, ReadonlyRootfs: !hardeningDrift, CapDrop: ["ALL"], SecurityOpt: ["no-new-privileges:true"], @@ -389,14 +387,20 @@ function dockerFixture( Status: container?.status ?? "created", }, NetworkSettings: { - Networks: { "nemoclaw-llama-cpp-internal": { NetworkID: networkId } }, + Networks: { + [bindings().network.name]: { + NetworkID: startedOnce ? networkId : "", + IPAddress: container?.running ? "172.30.0.2" : "", + }, + }, Ports: { - "8081/tcp": startedOnce - ? Array.from({ length: publishedBindingCount }, () => ({ - HostIp: publishedHostIp, - HostPort: effectivePublishedHostPort, - })) - : null, + "8081/tcp": + startedOnce && publishedBindingCount > 0 + ? Array.from({ length: publishedBindingCount }, () => ({ + HostIp: publishedHostIp, + HostPort: effectivePublishedHostPort, + })) + : null, }, }, Mounts: [ @@ -415,13 +419,28 @@ function dockerFixture( ], }, ]; - const capture = vi.fn((args: readonly string[]) => { const unexpected = `unexpected Docker command: ${args.join(" ")}`; switch (args[0]) { case "network": switch (args[1]) { case "inspect": + switch (args[2]) { + case "openshell-docker": + return { + status: 0, + stdout: JSON.stringify([ + { + Name: "openshell-docker", + Internal: false, + Driver: "bridge", + Scope: "local", + IPAM: { Config: [{ Subnet: "172.29.0.0/16", Gateway: "172.29.0.1" }] }, + }, + ]), + stderr: "", + }; + } switch (networkPresent) { case false: absentNetworkInspectHook?.(); @@ -609,6 +628,7 @@ function dockerFixture( networkId = journal.networkId; networkTransactionId = journal.transactionId; networkPresent = true; + startedOnce = journal.phase !== "creating" && journal.phase !== "created"; container = { labels: { "io.nvidia.nemoclaw.host-local-inference.managed": "true", @@ -653,7 +673,7 @@ function options( } function controller(fixture: DockerFixture, store = journalStore(), now: () => number = Date.now) { - return createDockerLlamaCppManagedLifecycle(options(fixture, store), { + return createLifecycle(options(fixture, store), { now, }); } @@ -716,15 +736,17 @@ function preparedJournal(): HostLocalCreateJournalRecord { describe("dormant Docker llama.cpp managed lifecycle", () => { it("journals a product install on its declared loopback host port (#8544)", () => { - const fixture = dockerFixture("8081"); + const fixture = dockerFixture(); const store = journalStore(); - const lifecycle = createDockerLlamaCppManagedLifecycle( + const privateBridge = privateBridgeFixture(); + const lifecycle = createLifecycle( options(fixture, store, { ...bindings(), hostPort: 8081 }), + {}, + privateBridge, ); const writer = receiptWriter(); const receipt = lifecycle.start(writer); const serialized = serializeHostLocalInferenceReceipt(receipt); - expect(receipt.endpoint.port).toBe(8081); expect(receipt.runtime).toMatchObject({ kind: "container", runtimeId: RUNTIME_ID, @@ -736,14 +758,16 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { networkId: NETWORK_ID, }); expect(writer.writeExact).toHaveBeenCalledExactlyOnceWith(serialized); + expect(privateBridge.start).toHaveBeenCalledWith( + expect.objectContaining({ + transactionId: TRANSACTION_ID, + targetHost: "172.30.0.2", + bindAddresses: ["127.0.0.1", "172.29.0.1"], + }), + ); expect(serialized).not.toContain(modelPath); - expect(serialized).not.toContain(apiKeyPath); expect(serialized).not.toContain("filesystemIdentity"); expect(serialized).not.toContain("test-only-secret"); - const roundTrip = serializeHostLocalInferenceReceipt( - parseHostLocalInferenceReceipt(serialized), - ); - expect(roundTrip).toBe(serialized); expect(lifecycle.runtime.inspectManaged(receipt).running).toBe(true); expect(lifecycle.runtime.stopManaged(receipt).running).toBe(false); expect(lifecycle.runtime.prepareDestroy(receipt)).toEqual(receipt); @@ -831,40 +855,21 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { expect(store.load(TRANSACTION_ID)).toMatchObject({ phase: "finalized", runtimeId: RUNTIME_ID }); }); it.each([ - ["configured", "8082", undefined, /bound host port/u], - ["published", "8081", "8082", /declared binding/u], - ] as const)("rolls back exact ownership for %s loopback port drift (#8544)", (_kind, configured, published, expectedError) => { - const [fixture, store] = [dockerFixture(configured, published), journalStore()]; - const lifecycle = createDockerLlamaCppManagedLifecycle( - options(fixture, store, { ...bindings(), hostPort: 8081 }), - ); + ["configured", "8081", undefined, "127.0.0.1", 0, /must not configure/u], + ["runtime", "", "8081", "127.0.0.1", 1, /must not publish/u], + ["runtime-wide", "", "8081", "0.0.0.0", 1, /must not publish/u], + ] as const)("rolls back exact ownership for unexpected %s Docker publication (#8544)", (_kind, configured, published, ip, count, expectedError) => { + const [fixture, store] = [dockerFixture(configured, published, ip, count), journalStore()]; + const lifecycle = createLifecycle(options(fixture, store)); expect(() => lifecycle.start(receiptWriter())).toThrow(expectedError); const calls = fixture.capture.mock.calls.map((call) => call[0]); expect(calls).toContainEqual(["rm", "--force", RUNTIME_ID]); expect(calls).toContainEqual(["network", "rm", NETWORK_ID]); expect(store.list()).toEqual([]); }); - it("rejects and cleans up malformed or non-loopback published bindings (#8544)", () => { - for (const args of [ - [HOST_PORT, HOST_PORT, "0.0.0.0", 1], - ["8081", "8082", "0.0.0.0", 1], - ["8081", "invalid", "127.0.0.1", 1], - ["8081", "8082", "127.0.0.1", 2], - ] as const) { - const [fixture, store] = [dockerFixture(args[0], args[1], args[2], args[3]), journalStore()]; - const lifecycle = createDockerLlamaCppManagedLifecycle( - options(fixture, store, { ...bindings(), hostPort: 8081 }), - ); - expect(() => lifecycle.start(receiptWriter())).toThrow(/port|binding/u); - const calls = fixture.capture.mock.calls.map((call) => call[0]); - expect(store.list()).toEqual([]); - expect(calls).toContainEqual(["rm", "--force", RUNTIME_ID]); - expect(calls).toContainEqual(["network", "rm", NETWORK_ID]); - } - }); it("uses the declarative readiness timeout as both curl retry budget and capture budget", () => { const fixture = dockerFixture(); - const lifecycle = createDockerLlamaCppManagedLifecycle({ + const lifecycle = createLifecycle({ ...options(fixture), readinessTimeoutSeconds: 37, }); @@ -888,7 +893,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { it("rejects an invalid declarative readiness timeout before inspection or mutation", () => { const fixture = dockerFixture(); expect(() => - createDockerLlamaCppManagedLifecycle({ + createLifecycle({ ...options(fixture), readinessTimeoutSeconds: 0, }), @@ -912,7 +917,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ...options(fixture, store), plan: { ...plan(), planDigest: `sha256:${"0".repeat(64)}` }, }; - expect(() => createDockerLlamaCppManagedLifecycle(invalid)).toThrow("canonical payload"); + expect(() => createLifecycle(invalid)).toThrow("canonical payload"); expect(store.list()).toEqual([]); expect(fixture.capture).not.toHaveBeenCalled(); }); @@ -942,7 +947,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ...changedPayload, planDigest: digest(changedPayload), }; - const lifecycle = createDockerLlamaCppManagedLifecycle({ + const lifecycle = createLifecycle({ ...options(fixture, store), plan: changedPlan, }); @@ -1093,6 +1098,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { switch (committed) { case null: committed = serializedReceipt; + fs.writeFileSync(path.join(apiKeyRoot, "receipt.json"), serializedReceipt); throw new Error("writer outcome unknown"); default: invariant(committed === serializedReceipt, "different receipt"); @@ -1310,10 +1316,9 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { persistedAuthority.record(journal.engineAuthority); fixture.setAbsentNetworkInspectError(stderr); - const recovery = createDockerLlamaCppManagedLifecycle( - options(fixture, store, bindings(), persistedAuthority), - { now: () => 31 * 60 * 1_000 }, - ).recoverUnfinished(receiptWriter()); + const recovery = createLifecycle(options(fixture, store, bindings(), persistedAuthority), { + now: () => 31 * 60 * 1_000, + }).recoverUnfinished(receiptWriter()); expect(recovery.recovered).toEqual([]); expect(recovery.failures[0]?.message).toContain("network inspection failed"); @@ -1336,10 +1341,9 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { fixture.setAbsentNetworkInspectError("network nemoclaw-llama-cpp-internal not found"); expect( - createDockerLlamaCppManagedLifecycle( - options(fixture, store, bindings(), persistedAuthority), - { now: () => 31 * 60 * 1_000 }, - ).recoverUnfinished(receiptWriter()), + createLifecycle(options(fixture, store, bindings(), persistedAuthority), { + now: () => 31 * 60 * 1_000, + }).recoverUnfinished(receiptWriter()), ).toEqual({ recovered: [TRANSACTION_ID], failures: [] }); expect(store.list()).toEqual([]); expect(dockerCommandPrefixes(fixture)).not.toContainEqual(["network", "rm"]); @@ -1368,10 +1372,9 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { arrangePhase[phase](); const persistedAuthority = authorityStore(); persistedAuthority.record(base.engineAuthority); - const recovery = createDockerLlamaCppManagedLifecycle( - options(fixture, store, bindings(), persistedAuthority), - { now: () => 31 * 60 * 1_000 }, - ).recoverUnfinished(receiptWriter()); + const recovery = createLifecycle(options(fixture, store, bindings(), persistedAuthority), { + now: () => 31 * 60 * 1_000, + }).recoverUnfinished(receiptWriter()); expect(recovery).toEqual({ recovered: [TRANSACTION_ID], failures: [] }); expect(store.list()).toEqual([]); } @@ -1396,7 +1399,7 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { }), } as const; arrangeAuthority[state](); - const recovery = createDockerLlamaCppManagedLifecycle( + const recovery = createLifecycle( options(fixture, store, bindings(), persistedAuthority), ).recoverUnfinished(receiptWriter()); expect(recovery.recovered).toEqual([]); @@ -1422,7 +1425,6 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { const receipt = lifecycle.start(receiptWriter()); fixture.driftHardening(); expect(() => lifecycle.runtime.inspectManaged(receipt)).toThrow("exact journal authority"); - for (const mutate of [ (candidate: DockerFixture) => candidate.driftGpuRequest(undefined, 1), (candidate: DockerFixture) => candidate.driftGpuRequest("nvidia", 2), @@ -1439,7 +1441,6 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { ); } }); - it("rejects model and API-key filesystem identity drift during exact inspection", () => { const modelFixture = dockerFixture(); const modelLifecycle = controller(modelFixture); @@ -1448,34 +1449,33 @@ describe("dormant Docker llama.cpp managed lifecycle", () => { expect(() => modelLifecycle.runtime.inspectManaged(modelReceipt)).toThrow( "filesystem identity", ); - fs.writeFileSync(modelPath, MODEL_CONTENT, { mode: 0o600 }); const keyFixture = dockerFixture(); const keyLifecycle = controller(keyFixture); - const keyReceipt = keyLifecycle.start(receiptWriter()); + const keyReceipt = keyLifecycle.start( + receiptWriter((serializedReceipt) => { + fs.writeFileSync(path.join(apiKeyRoot, "receipt.json"), serializedReceipt, { mode: 0o600 }); + return serializedReceipt; + }), + ); + expect(() => keyLifecycle.runtime.inspectManaged(keyReceipt)).not.toThrow(); fs.writeFileSync(apiKeyPath, "replacement-test-key\n", { mode: 0o600 }); expect(() => keyLifecycle.runtime.inspectManaged(keyReceipt)).toThrow("API-key identity"); }); - it("rejects a same-size GGUF replacement when inspection reconstructs current identity", () => { const fixture = dockerFixture(); const store = journalStore(); const persistedAuthority = authorityStore(); - const initial = createDockerLlamaCppManagedLifecycle( - options(fixture, store, bindings(), persistedAuthority), - ); + const initial = createLifecycle(options(fixture, store, bindings(), persistedAuthority)); const receipt = initial.start(receiptWriter()); - fs.writeFileSync(modelPath, Buffer.alloc(MODEL_CONTENT.length, 0x62)); - const currentIdentityInspector = createDockerLlamaCppManagedLifecycle( + const currentIdentityInspector = createLifecycle( options(fixture, store, bindings(), persistedAuthority), ); - expect(() => currentIdentityInspector.runtime.inspectManaged(receipt)).toThrow( "durable create journal", ); }); - it("fails closed on crafted absent destroy authority and status-one daemon errors (#8395)", () => { const fixture = dockerFixture(); const store = journalStore(); diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts index 23e387e2a39..4a857bb730b 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts @@ -21,6 +21,11 @@ import { type LlamaCppHostLocalLaunchContract, type LlamaCppHostLocalRuntimeBindings, } from "../../inference/llama-cpp/host-local-runtime"; +import { + createDockerLlamaCppPrivateBridgeController, + type DockerLlamaCppPrivateBridgeAuthority, + type DockerLlamaCppPrivateBridgeController, +} from "./docker-llama-cpp-private-bridge"; import { type HostLocalCreateJournalExecutionLease, type HostLocalCreateJournalRecord, @@ -51,6 +56,7 @@ import { const PROVIDER_ID = "docker"; const SERVICE = "llama-cpp"; const ENDPOINT_HOST = "host.openshell.internal"; +const OPENSHELL_DOCKER_NETWORK = "openshell-docker"; const FULL_ID = /^[a-f0-9]{64}$/u; const SHA256 = /^[a-f0-9]{64}$/u; const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; @@ -71,6 +77,7 @@ export type DockerLlamaCppManagedLifecycleOptions = HostLocalLlamaCppLifecycleIn export interface DockerLlamaCppManagedLifecycleDependencies { readonly now?: () => number; + readonly privateBridge?: DockerLlamaCppPrivateBridgeController; } export type DockerLlamaCppRecoveryResult = HostLocalInferenceRecoveryResult; @@ -82,6 +89,11 @@ interface DockerNetworkAuthority { readonly name: string; } +interface DockerGatewayBridgeAuthority { + readonly gatewayIp: string; + readonly name: typeof OPENSHELL_DOCKER_NETWORK; +} + interface DockerContainerInspection { readonly id: string; readonly name: string; @@ -89,9 +101,9 @@ interface DockerContainerInspection { readonly labels: Readonly>; readonly running: boolean; readonly status: string; - readonly networkId: string; + readonly networkId: string | null; readonly networkName: string; - readonly hostPort: number | null; + readonly containerIp: string | null; readonly mounts: readonly { readonly type: string; readonly source: string; @@ -118,7 +130,7 @@ interface DockerContainerInspection { }; } -type DockerContainerInspectionMode = "runtime" | "cleanup"; +type DockerContainerInspectionMode = "created" | "runtime" | "cleanup"; interface StableFileIdentity { readonly dev: bigint; @@ -199,12 +211,22 @@ function exactId(value: unknown, label: string): string { return value; } -function exactPort(value: unknown): number { - const port = typeof value === "string" && /^[0-9]{1,5}$/u.test(value) ? Number(value) : -1; - if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { - throw new Error("Docker llama.cpp inspection returned an invalid host port."); +function exactPrivateIpv4(value: unknown, label: string): string { + if (typeof value !== "string" || !/^[0-9]{1,3}(?:\.[0-9]{1,3}){3}$/u.test(value)) { + throw new Error(`${label} must be one private IPv4 address.`); + } + const octets = value.split(".").map(Number); + if ( + octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255) || + !( + octets[0] === 10 || + (octets[0] === 172 && octets[1]! >= 16 && octets[1]! <= 31) || + (octets[0] === 192 && octets[1] === 168) + ) + ) { + throw new Error(`${label} must be one private IPv4 address.`); } - return port; + return value; } function inspectNetworkIfPresent( @@ -250,6 +272,47 @@ function inspectNetworkIfPresent( }); } +function inspectGatewayBridge(engine: ContainerEngine): DockerGatewayBridgeAuthority { + const output = requireSuccess( + "OpenShell bridge inspection", + engine.capture(["network", "inspect", OPENSHELL_DOCKER_NETWORK], INSPECT_TIMEOUT_MS), + ); + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Docker llama.cpp OpenShell bridge inspection returned unreadable JSON."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Docker llama.cpp requires exactly one OpenShell Docker bridge."); + } + const source = record(parsed[0], "Docker llama.cpp OpenShell bridge inspection"); + const ipam = record(source.IPAM, "Docker llama.cpp OpenShell bridge IPAM"); + const configs = ipam.Config; + if ( + source.Name !== OPENSHELL_DOCKER_NETWORK || + source.Driver !== "bridge" || + source.Scope !== "local" || + source.Internal !== false || + !Array.isArray(configs) + ) { + throw new Error("Docker llama.cpp requires the native OpenShell Docker bridge."); + } + const gatewayIps = configs.flatMap((candidate) => { + const config = record(candidate, "Docker llama.cpp OpenShell bridge IPAM entry"); + return typeof config.Gateway === "string" && + /^172\.(?:1[6-9]|2[0-9]|3[01])\.(?:[0-9]{1,3})\.(?:[0-9]{1,3})$/u.test(config.Gateway) + ? [config.Gateway] + : typeof config.Gateway === "string" && /^(?:10\.|192\.168\.)/u.test(config.Gateway) + ? [config.Gateway] + : []; + }); + if (gatewayIps.length !== 1) { + throw new Error("Docker llama.cpp requires one private IPv4 OpenShell bridge gateway."); + } + return Object.freeze({ gatewayIp: gatewayIps[0]!, name: OPENSHELL_DOCKER_NETWORK }); +} + function inspectNetwork( engine: ContainerEngine, name: string, @@ -301,42 +364,35 @@ function parseInspection( throw new Error("Docker llama.cpp container has unexpected network attachments."); } const attached = record(networks[networkName], "Docker llama.cpp network attachment"); - let hostPort: number | null = null; - if (mode === "runtime") { + const attachedNetworkId = attached.NetworkID; + const networkId = + typeof attachedNetworkId === "string" && FULL_ID.test(attachedNetworkId) + ? attachedNetworkId + : mode !== "runtime" && attachedNetworkId === "" + ? null + : exactId(attachedNetworkId, "Docker attached network identity"); + const attachedIp = attached.IPAddress; + const containerIp = + typeof attachedIp === "string" && attachedIp !== "" + ? exactPrivateIpv4(attachedIp, "Docker llama.cpp container address") + : (mode !== "runtime" || state.Running === false) && attachedIp === "" + ? null + : exactPrivateIpv4(attachedIp, "Docker llama.cpp container address"); + if (mode !== "cleanup") { const ports = record(networkSettings.Ports, "Docker llama.cpp published ports"); const portKey = `${String(contract.serve.port)}/tcp`; const configuredPorts = record(hostConfig.PortBindings, "Docker llama.cpp configured ports"); - if (Object.keys(configuredPorts).length !== 1) { - throw new Error("Docker llama.cpp container has extra configured ports."); - } - const configuredBindings = configuredPorts[portKey]; - if (!Array.isArray(configuredBindings) || configuredBindings.length !== 1) { - throw new Error("Docker llama.cpp container has unexpected configured ports."); - } - const configuredPort = record(configuredBindings[0], "Docker llama.cpp configured port"); - if (configuredPort.HostIp !== "127.0.0.1") { - throw new Error("Docker llama.cpp configured host port is not loopback-only."); + if (Object.keys(configuredPorts).length !== 0) { + throw new Error("Docker llama.cpp container must not configure published ports."); } - if (configuredPort.HostPort !== String(bindings.hostPort)) { - throw new Error("Docker llama.cpp configured host port is not the bound host port."); - } - const publishedBindings = ports[portKey]; + const portKeys = Object.keys(ports); if ( - publishedBindings !== null && - (!Array.isArray(publishedBindings) || publishedBindings.length !== 1) + portKeys.some((key) => key !== portKey) || + (portKeys.includes(portKey) && ports[portKey] !== null) ) { - throw new Error("Docker llama.cpp container has unexpected published ports."); - } - const published = - publishedBindings === null - ? null - : record(publishedBindings[0], "Docker llama.cpp published port"); - if (published !== null && published.HostIp !== "127.0.0.1") { - throw new Error("Docker llama.cpp host port is not loopback-only."); - } - hostPort = published === null ? null : exactPort(published.HostPort); - if (hostPort !== null && hostPort !== bindings.hostPort) { - throw new Error("Docker llama.cpp published host port differs from its declared binding."); + throw new Error( + "Docker llama.cpp container must not publish ports from its internal network.", + ); } } if (!Array.isArray(source.Mounts)) { @@ -404,9 +460,9 @@ function parseInspection( labels: parseLabels(config.Labels), running: state.Running, status: stateStatus, - networkId: exactId(attached.NetworkID, "Docker attached network identity"), + networkId, networkName, - hostPort, + containerIp, mounts: Object.freeze(mounts), hardening: Object.freeze({ user: String(config.User ?? ""), @@ -658,6 +714,16 @@ function assertApiKeyIdentity( } } +function assertApiKeyFileIdentity( + options: DockerLlamaCppManagedLifecycleOptions, + expected: StableFileIdentity, +): void { + // Receipt publication may change the parent timestamp after Docker has bound the exact key inode. + if (!sameFileIdentity(apiKeyIdentity(options), expected)) { + throw new Error("Docker llama.cpp API-key file changed during lifecycle mutation."); + } +} + function qualifyEngine(options: DockerLlamaCppManagedLifecycleOptions): PersistedEngineAuthority { if ( options.engine.operation !== "host-local-inference" || @@ -782,7 +848,9 @@ function requireOwnedContainer( (record.runtimeId !== null && container.id !== record.runtimeId) || container.name !== record.containerName || container.imageRef !== options.bindings.imageReference || - container.networkId !== record.networkId || + (container.networkId === null + ? container.running || container.status !== "created" + : container.networkId !== record.networkId) || container.networkName !== options.bindings.network.name || container.labels[MANAGED_LABEL] !== "true" || container.labels[PROVIDER_LABEL] !== PROVIDER_ID || @@ -875,13 +943,117 @@ function probeReady( ); } +function privateBridgeAuthority( + options: DockerLlamaCppManagedLifecycleOptions, + journal: HostLocalCreateJournalRecord, + container: DockerContainerInspection, + gateway: DockerGatewayBridgeAuthority, +): DockerLlamaCppPrivateBridgeAuthority { + if (container.containerIp === null) { + throw new Error("Docker llama.cpp running container lacks an internal address."); + } + if (options.bindings.hostPort !== options.contract.serve.port) { + throw new Error("Docker llama.cpp host bridge port differs from its declarative server port."); + } + return Object.freeze({ + transactionId: journal.transactionId, + targetHost: container.containerIp, + targetPort: options.contract.serve.port, + listenPort: options.bindings.hostPort, + bindAddresses: Object.freeze(["127.0.0.1", gateway.gatewayIp]) as readonly [ + "127.0.0.1", + string, + ], + }); +} + +function probePrivateBridge( + options: DockerLlamaCppManagedLifecycleOptions, + bridge: DockerLlamaCppPrivateBridgeController, + journal: HostLocalCreateJournalRecord, + container: DockerContainerInspection, + lease: HostLocalCreateJournalExecutionLease, + execution: MutationExecutionState, +): void { + const gateway = inspectGatewayBridge(options.engine); + const authority = privateBridgeAuthority(options, journal, container, gateway); + options.journalStore.assertExecution(lease); + bridge.start(authority); + options.journalStore.assertExecution(lease); + const timeoutSeconds = Math.min(readinessTimeoutSeconds(options), 30); + const curlArguments = (url: string): readonly string[] => [ + "--fail", + "--silent", + "--show-error", + "--max-time", + String(timeoutSeconds), + "--retry", + String(timeoutSeconds), + "--retry-delay", + "1", + "--retry-max-time", + String(timeoutSeconds), + "--retry-connrefused", + url, + ]; + bridge.assertRunning(authority); + options.journalStore.assertExecution(lease); + requireSuccess( + "private loopback bridge probe", + captureMutation( + options, + lease, + execution, + [ + "run", + "--rm", + "--pull=never", + "--network", + "host", + "--entrypoint", + "curl", + options.probeImageReference, + ...curlArguments(`http://127.0.0.1:${String(options.bindings.hostPort)}/health`), + ], + timeoutSeconds * 1_000 + INSPECT_TIMEOUT_MS, + ), + ); + requireSuccess( + "private sandbox bridge probe", + captureMutation( + options, + lease, + execution, + [ + "run", + "--rm", + "--pull=never", + "--network", + gateway.name, + "--add-host", + `${ENDPOINT_HOST}:${gateway.gatewayIp}`, + "--entrypoint", + "curl", + options.probeImageReference, + ...curlArguments(`http://${ENDPOINT_HOST}:${String(options.bindings.hostPort)}/health`), + ], + timeoutSeconds * 1_000 + INSPECT_TIMEOUT_MS, + ), + ); + bridge.assertRunning(authority); + options.journalStore.assertExecution(lease); +} + function rollbackExact( options: DockerLlamaCppManagedLifecycleOptions, + bridge: DockerLlamaCppPrivateBridgeController, record: HostLocalCreateJournalRecord, lease: HostLocalCreateJournalExecutionLease, execution: MutationExecutionState, uncertainRecoveryUnixMs?: number, ): void { + options.journalStore.assertExecution(lease); + bridge.stopTransaction(record.transactionId); options.journalStore.assertExecution(lease); if (record.phase === "network-creating") { let network = inspectNetworkIfPresent( @@ -1024,8 +1196,8 @@ function receiptFor( journal: HostLocalCreateJournalRecord, container: DockerContainerInspection, ): HostLocalInferenceReceipt { - if (container.hostPort === null) { - throw new Error("Docker llama.cpp did not publish one loopback host port."); + if (!container.running || container.containerIp === null) { + throw new Error("Docker llama.cpp cannot publish a receipt for a stopped runtime."); } return normalizeHostLocalInferenceReceipt({ schemaVersion: 1, @@ -1034,7 +1206,7 @@ function receiptFor( engineAuthority: authority, endpoint: { host: ENDPOINT_HOST, - port: container.hostPort, + port: options.bindings.hostPort, networkName: options.bindings.network.name, }, runtime: { @@ -1122,6 +1294,7 @@ export function createDockerLlamaCppManagedLifecycle( normalizeHostLocalInferenceImageRef(options.probeImageReference); readinessTimeoutSeconds(options); const qualifiedAuthority = qualifyEngine(options); + const privateBridge = dependencies.privateBridge ?? createDockerLlamaCppPrivateBridgeController(); const authorizeStaticReceipt = (value: HostLocalInferenceReceipt) => { const receipt = normalizeHostLocalInferenceReceipt(value); @@ -1229,7 +1402,7 @@ export function createDockerLlamaCppManagedLifecycle( if (inspected === null) throw new Error("Docker llama.cpp owned runtime is absent."); const container = requireOwnedContainer(inspected, options, authorized.journal); if ( - container.hostPort !== authorized.receipt.endpoint.port || + options.bindings.hostPort !== authorized.receipt.endpoint.port || authorized.receipt.endpoint.host !== ENDPOINT_HOST || authorized.receipt.endpoint.networkName !== options.bindings.network.name ) { @@ -1258,8 +1431,20 @@ export function createDockerLlamaCppManagedLifecycle( if (apiKeyIdentitySha256(activeKeyIdentity) !== inspected.journal.apiKeyIdentitySha256) { throw new Error("Docker llama.cpp API-key identity differs from its create journal."); } - assertApiKeyIdentity(options, activeKeyIdentity, inspected.journal.apiKeyRootIdentitySha256); + assertApiKeyFileIdentity(options, activeKeyIdentity); assertModelFilesystemAuthority(options); + if (inspected.container.running) { + privateBridge.assertRunning( + privateBridgeAuthority( + options, + inspected.journal, + inspected.container, + inspectGatewayBridge(options.engine), + ), + ); + } else { + privateBridge.assertStopped(inspected.journal.transactionId); + } return Object.freeze({ running: inspected.container.running, receipt: inspected.receipt, @@ -1271,6 +1456,10 @@ export function createDockerLlamaCppManagedLifecycle( const execution: MutationExecutionState = { unknown: false }; try { let inspected = inspectAuthorized(receipt); + options.journalStore.assertExecution(lease); + privateBridge.stopTransaction(inspected.journal.transactionId); + privateBridge.assertStopped(inspected.journal.transactionId); + options.journalStore.assertExecution(lease); if (!inspected.container.running) { if (!AT_REST.has(inspected.container.status)) { throw new Error("Docker llama.cpp container is not in an exact stoppable state."); @@ -1307,22 +1496,22 @@ export function createDockerLlamaCppManagedLifecycle( throw new Error("Docker llama.cpp API-key identity differs from its create journal."); } assertModelFilesystemAuthority(options); - assertApiKeyIdentity( - options, - activeKeyIdentity, - authorized.journal.apiKeyRootIdentitySha256, - ); + assertApiKeyFileIdentity(options, activeKeyIdentity); const inspected = inspectAuthorized(receipt); if (!inspected.container.running) { throw new Error("Docker llama.cpp cannot preserve a stopped runtime."); } probeReady(options, lease, execution); - assertModelFilesystemAuthority(options); - assertApiKeyIdentity( + probePrivateBridge( options, - activeKeyIdentity, - authorized.journal.apiKeyRootIdentitySha256, + privateBridge, + inspected.journal, + inspected.container, + lease, + execution, ); + assertModelFilesystemAuthority(options); + assertApiKeyFileIdentity(options, activeKeyIdentity); requireExactNetwork( options, requireJournalNetworkId(inspected.journal), @@ -1359,12 +1548,16 @@ export function createDockerLlamaCppManagedLifecycle( options.bindings, ); if (existing === null) { + privateBridge.stopTransaction(normalized.runtime.model.generation); + privateBridge.assertStopped(normalized.runtime.model.generation); const journal = options.journalStore.load(normalized.runtime.model.generation); if (journal !== null) { const lease = options.journalStore.acquireExecution(journal.transactionId); try { const authorized = authorizeReceipt(normalized, true); options.journalStore.assertExecution(lease); + privateBridge.stopTransaction(authorized.journal.transactionId); + options.journalStore.assertExecution(lease); options.journalStore.retire(authorized.journal.transactionId); options.journalStore.assertExecution(lease); } finally { @@ -1380,6 +1573,9 @@ export function createDockerLlamaCppManagedLifecycle( const execution: MutationExecutionState = { unknown: false }; try { const inspected = inspectAuthorized(normalized); + options.journalStore.assertExecution(lease); + privateBridge.stopTransaction(inspected.journal.transactionId); + options.journalStore.assertExecution(lease); requireSuccess( "container removal", captureMutation( @@ -1426,11 +1622,7 @@ export function createDockerLlamaCppManagedLifecycle( throw new Error("Docker llama.cpp API-key identity differs from its create journal."); } assertModelFilesystemAuthority(options); - assertApiKeyIdentity( - options, - activeKeyIdentity, - authorized.journal.apiKeyRootIdentitySha256, - ); + assertApiKeyFileIdentity(options, activeKeyIdentity); let inspected = inspectAuthorized(receipt); if (!inspected.container.running) { if (!AT_REST.has(inspected.container.status)) { @@ -1452,12 +1644,16 @@ export function createDockerLlamaCppManagedLifecycle( } } probeReady(options, lease, execution); - assertModelFilesystemAuthority(options); - assertApiKeyIdentity( + probePrivateBridge( options, - activeKeyIdentity, - authorized.journal.apiKeyRootIdentitySha256, + privateBridge, + inspected.journal, + inspected.container, + lease, + execution, ); + assertModelFilesystemAuthority(options); + assertApiKeyFileIdentity(options, activeKeyIdentity); requireExactNetwork( options, requireJournalNetworkId(authorized.journal), @@ -1573,6 +1769,7 @@ export function createDockerLlamaCppManagedLifecycle( options.bindings.containerName, options.contract, options.bindings, + "created", ); if (create.error || create.status !== 0 || created === null) { throw new Error( @@ -1611,6 +1808,7 @@ export function createDockerLlamaCppManagedLifecycle( assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); requireExactNetwork(options, network.id, transactionId); probeReady(options, lease, execution); + probePrivateBridge(options, privateBridge, journal, started, lease, execution); assertModelFilesystemAuthority(options); assertApiKeyIdentity(options, startingKeyIdentity, startingApiKeyRootIdentitySha256); requireExactNetwork(options, network.id, transactionId); @@ -1650,7 +1848,7 @@ export function createDockerLlamaCppManagedLifecycle( !execution.unknown ) { try { - rollbackExact(options, journal, lease, execution); + rollbackExact(options, privateBridge, journal, lease, execution); } catch (rollbackError) { rollbackFailure = rollbackError; } @@ -1728,14 +1926,22 @@ export function createDockerLlamaCppManagedLifecycle( throw new Error("Docker llama.cpp API-key identity differs from its create journal."); } assertModelFilesystemAuthority(options); - assertApiKeyIdentity(options, activeKeyIdentity, journal.apiKeyRootIdentitySha256); + assertApiKeyFileIdentity(options, activeKeyIdentity); const inspected = inspectAuthorized(receipt, false); if (!inspected.container.running) { throw new Error("Docker llama.cpp receipt publication requires a running runtime."); } probeReady(options, lease, execution); + probePrivateBridge( + options, + privateBridge, + journal, + inspected.container, + lease, + execution, + ); assertModelFilesystemAuthority(options); - assertApiKeyIdentity(options, activeKeyIdentity, journal.apiKeyRootIdentitySha256); + assertApiKeyFileIdentity(options, activeKeyIdentity); requireExactNetwork(options, requireJournalNetworkId(journal), journal.transactionId); options.journalStore.assertExecution(lease); writePreparedReceipt(writer, journal); @@ -1745,6 +1951,7 @@ export function createDockerLlamaCppManagedLifecycle( } else { rollbackExact( options, + privateBridge, journal, lease, execution, diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge-process.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge-process.ts new file mode 100644 index 00000000000..7b382578dee --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge-process.ts @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; + +const SHA256 = /^[a-f0-9]{64}$/u; + +export interface LlamaCppPrivateBridgeArguments { + readonly transactionId: string; + readonly targetHost: string; + readonly targetPort: number; + readonly listenPort: number; + readonly bindAddresses: readonly ["127.0.0.1", string]; +} + +function exactPort(value: string, label: string): number { + const port = /^[0-9]{1,5}$/u.test(value) ? Number(value) : -1; + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error(`${label} is invalid`); + } + return port; +} + +function isPrivateIpv4(value: string): boolean { + if (!net.isIPv4(value)) return false; + const [first, second] = value.split(".").map(Number); + return ( + first === 10 || + (first === 172 && second! >= 16 && second! <= 31) || + (first === 192 && second === 168) + ); +} + +export function parseLlamaCppPrivateBridgeArguments( + argv: readonly string[], +): LlamaCppPrivateBridgeArguments { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) { + throw new Error("private bridge arguments must be exact key/value pairs"); + } + values.set(key, [...(values.get(key) ?? []), value]); + } + const one = (key: string): string => { + const candidates = values.get(key); + if (!candidates || candidates.length !== 1) throw new Error(`${key} must be provided once`); + return candidates[0]!; + }; + const transactionId = one("--transaction"); + const targetHost = one("--target-host"); + const bindAddresses = values.get("--bind-address") ?? []; + const supported = new Set([ + "--transaction", + "--target-host", + "--target-port", + "--listen-port", + "--bind-address", + ]); + if ([...values.keys()].some((key) => !supported.has(key))) { + throw new Error("private bridge received an unsupported argument"); + } + if ( + !SHA256.test(transactionId) || + !isPrivateIpv4(targetHost) || + bindAddresses.length !== 2 || + bindAddresses[0] !== "127.0.0.1" || + !isPrivateIpv4(bindAddresses[1]!) || + bindAddresses[1] === targetHost + ) { + throw new Error("private bridge authority is invalid"); + } + return Object.freeze({ + transactionId, + targetHost, + targetPort: exactPort(one("--target-port"), "target port"), + listenPort: exactPort(one("--listen-port"), "listen port"), + bindAddresses: Object.freeze(["127.0.0.1", bindAddresses[1]!]) as readonly [ + "127.0.0.1", + string, + ], + }); +} + +export async function runLlamaCppPrivateBridge( + authority: LlamaCppPrivateBridgeArguments, +): Promise { + const servers = authority.bindAddresses.map((host) => + net.createServer({ allowHalfOpen: false, pauseOnConnect: true }, (client) => { + const upstream = net.createConnection({ + host: authority.targetHost, + port: authority.targetPort, + }); + const close = () => { + client.destroy(); + upstream.destroy(); + }; + client.on("error", close); + upstream.on("error", close); + upstream.once("connect", () => { + client.pipe(upstream); + upstream.pipe(client); + client.resume(); + }); + }), + ); + + const close = () => { + for (const server of servers) server.close(); + }; + process.once("SIGINT", close); + process.once("SIGTERM", close); + + await Promise.all( + servers.map( + (server, index) => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen( + { host: authority.bindAddresses[index]!, port: authority.listenPort, exclusive: true }, + () => resolve(), + ); + }), + ), + ); + await new Promise((_resolve, reject) => { + for (const server of servers) server.once("error", reject); + }); +} + +if (require.main === module) { + runLlamaCppPrivateBridge(parseLlamaCppPrivateBridgeArguments(process.argv.slice(2))).catch( + (error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }, + ); +} diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test-support.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test-support.ts new file mode 100644 index 00000000000..111ee79aae9 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test-support.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; + +import { + createDockerLlamaCppManagedLifecycle, + type DockerLlamaCppManagedLifecycleOptions, +} from "./docker-llama-cpp-managed-lifecycle"; +import { invariant } from "./docker-llama-cpp-managed-lifecycle.test-support"; + +export function privateBridgeFixture() { + let active: string | null = null; + return { + start: vi.fn((authority: unknown) => (active = JSON.stringify(authority))), + assertRunning: vi.fn((authority: unknown) => + invariant(active === JSON.stringify(authority), "bridge is not running"), + ), + assertStopped: vi.fn(() => invariant(active === null, "bridge is still running")), + stopTransaction: vi.fn(() => (active = null)), + }; +} + +export function createTestDockerLlamaCppManagedLifecycle( + lifecycleOptions: DockerLlamaCppManagedLifecycleOptions, + dependencies: { readonly now?: () => number } = {}, + privateBridge = privateBridgeFixture(), +) { + return createDockerLlamaCppManagedLifecycle(lifecycleOptions, { + ...dependencies, + privateBridge, + }); +} diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test.ts new file mode 100644 index 00000000000..34a9b50ecf6 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcess, spawn } from "node:child_process"; + +import { describe, expect, it, vi } from "vitest"; +import { + createDockerLlamaCppPrivateBridgeController, + type DockerLlamaCppPrivateBridgeAuthority, +} from "./docker-llama-cpp-private-bridge"; +import { parseLlamaCppPrivateBridgeArguments } from "./docker-llama-cpp-private-bridge-process"; + +const TRANSACTION = "9".repeat(64); +const authority: DockerLlamaCppPrivateBridgeAuthority = { + transactionId: TRANSACTION, + targetHost: "172.30.0.2", + targetPort: 8081, + listenPort: 8081, + bindAddresses: ["127.0.0.1", "172.29.0.1"], +}; + +function fixture() { + let nextPid = 40_001; + const processes = new Map(); + const signals: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const spawnProcess = vi.fn((file: string, args: readonly string[]) => { + const pid = nextPid++; + processes.set(pid, [file, ...args]); + return { pid, unref: vi.fn() } as unknown as ChildProcess; + }) as unknown as typeof spawn; + const controller = createDockerLlamaCppPrivateBridgeController({ + spawnProcess, + processIsAlive: (pid) => processes.has(pid), + signalProcess: (pid, signal) => { + signals.push({ pid, signal }); + processes.delete(pid); + }, + listProcessIds: () => [...processes.keys()], + readProcessArgv: (pid) => processes.get(pid) ?? null, + sleep: vi.fn(), + }); + return { controller, processes, signals, spawnProcess }; +} + +describe("Docker llama.cpp private bridge controller", () => { + it("owns one exact transaction-scoped bridge and stops only that process", () => { + const { controller, processes, signals, spawnProcess } = fixture(); + controller.start(authority); + controller.assertRunning(authority); + expect(spawnProcess).toHaveBeenCalledOnce(); + expect(spawnProcess).toHaveBeenCalledWith( + process.execPath, + expect.arrayContaining([ + expect.stringMatching(/docker-llama-cpp-private-bridge-process\.js$/u), + "--transaction", + TRANSACTION, + ]), + expect.objectContaining({ detached: true, env: {}, shell: false, stdio: "ignore" }), + ); + expect([...processes.values()][0]).toEqual( + expect.arrayContaining([ + "--transaction", + TRANSACTION, + "--target-host", + "172.30.0.2", + "--bind-address", + "127.0.0.1", + "--bind-address", + "172.29.0.1", + ]), + ); + controller.stopTransaction(TRANSACTION); + controller.assertStopped(TRANSACTION); + expect(signals).toEqual([{ pid: 40_001, signal: "SIGTERM" }]); + }); + + it("replaces drifted authority for the same transaction without touching another process", () => { + const { controller, processes, signals } = fixture(); + controller.start(authority); + const unrelated = [...processes.values()][0]!.slice(); + unrelated[3] = "8".repeat(64); + processes.set(50_000, unrelated); + controller.start({ ...authority, targetHost: "172.30.0.3" }); + expect(signals).toEqual([{ pid: 40_001, signal: "SIGTERM" }]); + expect(processes.has(50_000)).toBe(true); + controller.assertRunning({ ...authority, targetHost: "172.30.0.3" }); + }); + + it("fails closed when exact bridge ownership is ambiguous", () => { + const { controller, processes } = fixture(); + controller.start(authority); + processes.set(50_000, [...processes.values()][0]!); + expect(() => controller.assertRunning(authority)).toThrow("2 matching processes"); + }); +}); + +describe("llama.cpp private bridge argument boundary", () => { + const argv = [ + "--transaction", + TRANSACTION, + "--target-host", + "172.30.0.2", + "--target-port", + "8081", + "--listen-port", + "8081", + "--bind-address", + "127.0.0.1", + "--bind-address", + "172.29.0.1", + ]; + + it("accepts only the exact private loopback and OpenShell bridge topology", () => { + expect(parseLlamaCppPrivateBridgeArguments(argv)).toEqual(authority); + const publicTarget = argv.slice(); + publicTarget[3] = "8.8.8.8"; + expect(() => parseLlamaCppPrivateBridgeArguments(publicTarget)).toThrow("authority is invalid"); + const broadListener = argv.slice(); + broadListener[11] = "0.0.0.0"; + expect(() => parseLlamaCppPrivateBridgeArguments(broadListener)).toThrow( + "authority is invalid", + ); + }); +}); diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.ts new file mode 100644 index 00000000000..15c021e69cf --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.ts @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +const SHA256 = /^[a-f0-9]{64}$/u; +const PROCESS_EXIT_WAIT_MS = 5_000; +const PROCESS_EXIT_POLL_MS = 50; +const SLEEP_ARRAY = new Int32Array(new SharedArrayBuffer(4)); + +export interface DockerLlamaCppPrivateBridgeAuthority { + readonly transactionId: string; + readonly targetHost: string; + readonly targetPort: number; + readonly listenPort: number; + readonly bindAddresses: readonly ["127.0.0.1", string]; +} + +export interface DockerLlamaCppPrivateBridgeController { + start(authority: DockerLlamaCppPrivateBridgeAuthority): void; + assertRunning(authority: DockerLlamaCppPrivateBridgeAuthority): void; + assertStopped(transactionId: string): void; + stopTransaction(transactionId: string): void; +} + +export interface DockerLlamaCppPrivateBridgeDependencies { + readonly spawnProcess?: typeof spawn; + readonly processIsAlive?: (pid: number) => boolean; + readonly signalProcess?: (pid: number, signal: NodeJS.Signals) => void; + readonly listProcessIds?: () => readonly number[]; + readonly readProcessArgv?: (pid: number) => readonly string[] | null; + readonly sleep?: (milliseconds: number) => void; +} + +function exactPort(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value < 1 || value > 65_535) { + throw new Error(`Docker llama.cpp private bridge ${label} is invalid.`); + } + return value; +} + +function isPrivateIpv4(value: string): boolean { + if (!net.isIPv4(value)) return false; + const [first, second] = value.split(".").map(Number); + return ( + first === 10 || + (first === 172 && second! >= 16 && second! <= 31) || + (first === 192 && second === 168) + ); +} + +function normalizeAuthority( + value: DockerLlamaCppPrivateBridgeAuthority, +): DockerLlamaCppPrivateBridgeAuthority { + if ( + !SHA256.test(value.transactionId) || + !isPrivateIpv4(value.targetHost) || + value.bindAddresses.length !== 2 || + value.bindAddresses[0] !== "127.0.0.1" || + !isPrivateIpv4(value.bindAddresses[1]) || + value.bindAddresses[1] === value.targetHost + ) { + throw new Error("Docker llama.cpp private bridge authority is invalid."); + } + return Object.freeze({ + transactionId: value.transactionId, + targetHost: value.targetHost, + targetPort: exactPort(value.targetPort, "target port"), + listenPort: exactPort(value.listenPort, "listen port"), + bindAddresses: Object.freeze([...value.bindAddresses]) as readonly ["127.0.0.1", string], + }); +} + +function bridgeArguments(authorityValue: DockerLlamaCppPrivateBridgeAuthority): readonly string[] { + const authority = normalizeAuthority(authorityValue); + return Object.freeze([ + "--transaction", + authority.transactionId, + "--target-host", + authority.targetHost, + "--target-port", + String(authority.targetPort), + "--listen-port", + String(authority.listenPort), + "--bind-address", + authority.bindAddresses[0], + "--bind-address", + authority.bindAddresses[1], + ]); +} + +function defaultProcessIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function defaultSignalProcess(pid: number, signal: NodeJS.Signals): void { + process.kill(pid, signal); +} + +function defaultListProcessIds(): readonly number[] { + if (process.platform !== "linux") { + throw new Error("Docker llama.cpp private bridge requires a native Linux host."); + } + return fs + .readdirSync("/proc", { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && /^[0-9]+$/u.test(entry.name)) + .map((entry) => Number(entry.name)); +} + +function defaultReadProcessArgv(pid: number): readonly string[] | null { + try { + const value = fs.readFileSync(`/proc/${String(pid)}/cmdline`); + const argv = value.toString("utf8").split("\0"); + if (argv.at(-1) === "") argv.pop(); + return argv.length > 0 ? Object.freeze(argv) : null; + } catch { + return null; + } +} + +function exactArgv(left: readonly string[] | null, right: readonly string[]): boolean { + return ( + left !== null && left.length === right.length && left.every((value, i) => value === right[i]) + ); +} + +function defaultSleep(milliseconds: number): void { + Atomics.wait(SLEEP_ARRAY, 0, 0, milliseconds); +} + +export function createDockerLlamaCppPrivateBridgeController( + dependencies: DockerLlamaCppPrivateBridgeDependencies = {}, +): DockerLlamaCppPrivateBridgeController { + const scriptPath = path.join(__dirname, "docker-llama-cpp-private-bridge-process.js"); + const spawnProcess = dependencies.spawnProcess ?? spawn; + const processIsAlive = dependencies.processIsAlive ?? defaultProcessIsAlive; + const signalProcess = dependencies.signalProcess ?? defaultSignalProcess; + const listProcessIds = dependencies.listProcessIds ?? defaultListProcessIds; + const readProcessArgv = dependencies.readProcessArgv ?? defaultReadProcessArgv; + const sleep = dependencies.sleep ?? defaultSleep; + + const expectedArgv = (authority: DockerLlamaCppPrivateBridgeAuthority) => + Object.freeze([process.execPath, scriptPath, ...bridgeArguments(authority)]); + + const matchingProcessIds = ( + authority: DockerLlamaCppPrivateBridgeAuthority, + ): readonly number[] => { + const expected = expectedArgv(authority); + return Object.freeze( + listProcessIds() + .filter((pid) => processIsAlive(pid) && exactArgv(readProcessArgv(pid), expected)) + .sort((left, right) => left - right), + ); + }; + + const transactionProcessIds = (transactionId: string): readonly number[] => { + if (!SHA256.test(transactionId)) { + throw new Error("Docker llama.cpp private bridge transaction is invalid."); + } + return Object.freeze( + listProcessIds() + .filter((pid) => { + if (!processIsAlive(pid)) return false; + const argv = readProcessArgv(pid); + return ( + argv !== null && + argv[0] === process.execPath && + argv[1] === scriptPath && + argv[2] === "--transaction" && + argv[3] === transactionId + ); + }) + .sort((left, right) => left - right), + ); + }; + + const requireOne = (authority: DockerLlamaCppPrivateBridgeAuthority): number => { + const matches = matchingProcessIds(authority); + if (matches.length !== 1) { + throw new Error( + `Docker llama.cpp private bridge has ${String(matches.length)} matching processes; expected one.`, + ); + } + return matches[0]!; + }; + + const stopProcessIds = (matches: readonly number[]): void => { + for (const pid of matches) { + try { + signalProcess(pid, "SIGTERM"); + } catch { + // A concurrently exited exact process is already stopped. + } + } + const deadline = Date.now() + PROCESS_EXIT_WAIT_MS; + for (const pid of matches) { + while (processIsAlive(pid) && Date.now() < deadline) sleep(PROCESS_EXIT_POLL_MS); + if (processIsAlive(pid)) { + try { + signalProcess(pid, "SIGKILL"); + } catch { + // A concurrently exited exact process is already stopped. + } + } + } + }; + + return Object.freeze({ + start(authorityValue: DockerLlamaCppPrivateBridgeAuthority) { + const authority = normalizeAuthority(authorityValue); + const existing = matchingProcessIds(authority); + if (existing.length > 1) { + throw new Error("Docker llama.cpp private bridge ownership is ambiguous."); + } + if (existing.length === 1) { + return; + } + const stale = transactionProcessIds(authority.transactionId); + if (stale.length > 0) stopProcessIds(stale); + const child: ChildProcess = spawnProcess( + process.execPath, + [scriptPath, ...bridgeArguments(authority)], + { + detached: true, + stdio: "ignore", + shell: false, + env: {}, + }, + ); + if (!Number.isInteger(child.pid) || !child.pid || child.pid < 1) { + throw new Error("Docker llama.cpp private bridge did not return a process identity."); + } + child.unref(); + }, + assertRunning(authorityValue: DockerLlamaCppPrivateBridgeAuthority) { + requireOne(normalizeAuthority(authorityValue)); + }, + assertStopped(transactionId: string) { + if (transactionProcessIds(transactionId).length !== 0) { + throw new Error("Docker llama.cpp private bridge remained active while stopped."); + } + }, + stopTransaction(transactionId: string) { + stopProcessIds(transactionProcessIds(transactionId)); + }, + }); +} diff --git a/test/e2e/live/gpu-e2e-helpers.ts b/test/e2e/live/gpu-e2e-helpers.ts index 026d2be622d..d6043c9b466 100644 --- a/test/e2e/live/gpu-e2e-helpers.ts +++ b/test/e2e/live/gpu-e2e-helpers.ts @@ -21,6 +21,37 @@ const DEFAULT_GPU_E2E_MODEL = "qwen3.5:9b"; validateSandboxName(SANDBOX_NAME); export const PROXY_PORT = tcpPort(process.env.NEMOCLAW_OLLAMA_PROXY_PORT, "11435"); +export function shouldBootstrapLlamaCppGenericGpuTarget( + environment: NodeJS.ProcessEnv = process.env, +): boolean { + return ( + environment.NEMOCLAW_RUN_LIVE_E2E === "1" && + /^[a-f0-9]{40}$/u.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "") && + environment.E2E_LLAMA_CPP_DEDICATED_LANE !== "1" + ); +} + +export function buildLlamaCppCompatibilityTargetEnv( + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const forwarded = [ + "NEMOCLAW_E2E_CORRELATION_ID", + "NEMOCLAW_E2E_EXPECTED_SHA", + "NEMOCLAW_E2E_SHARD", + ].reduce( + (selected, key) => ({ + ...selected, + ...(base[key] === undefined ? {} : { [key]: base[key] }), + }), + {}, + ); + return { + ...buildAvailabilityProbeEnv(base), + ...forwarded, + NEMOCLAW_RUN_LIVE_E2E: "1", + }; +} + function tcpPort(value: string | undefined, fallback: string): string { const raw = value ?? fallback; if (!/^[1-9][0-9]*$/u.test(raw)) throw new Error(`invalid TCP port: ${raw}`); diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 5180ae2516c..061247ab221 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -9,6 +9,7 @@ import { assertAgentExecutionSucceeded, assertGpuInstallProofs, assertNvidiaAvailable, + buildLlamaCppCompatibilityTargetEnv, CLI, chatContent, cleanupGpu, @@ -25,9 +26,55 @@ import { readTokenFileChecked, restartProxy, SANDBOX_NAME, + shouldBootstrapLlamaCppGenericGpuTarget, } from "./gpu-e2e-helpers.ts"; const TIMEOUT_MS = 75 * 60_000; +const bootstrapLlamaCppTarget = shouldBootstrapLlamaCppGenericGpuTarget(); +type SkipTest = (reason?: string) => never; +const skipOllamaForLlamaCppCompatibility: (skip: SkipTest) => void = bootstrapLlamaCppTarget + ? (skip) => skip("dedicated llama.cpp target owns this pre-merge GPU run") + : () => undefined; + +// Manual PR E2E executes trusted main workflow YAML, so the new dedicated job +// cannot select its candidate test until this change lands. The existing GPU +// lane invokes that test for this revision only; the landed workflow marker +// keeps later PRs on the two independent dedicated lanes. +const llamaCppCompatibilityTest = bootstrapLlamaCppTarget ? test : test.skip; + +llamaCppCompatibilityTest( + "trusted pre-merge GPU lane exercises the dedicated managed llama.cpp target", + { + timeout: 85 * 60_000, + meta: { + e2ePhases: [ + "prepare the trusted pre-merge GPU bridge", + "run the dedicated managed llama.cpp live target", + ], + }, + }, + async ({ host, progress }) => { + progress.phase("prepare the trusted pre-merge GPU bridge"); + progress.phase("run the dedicated managed llama.cpp live target"); + const result = await host.command( + "npx", + [ + "tsx", + "tools/e2e/live-vitest-invocation.mts", + "run", + "--test-path", + "test/e2e/live/llama-cpp-generic-gpu.test.ts", + ], + { + artifactName: "llama-cpp-generic-gpu-compatibility-target", + cwd: REPO_ROOT, + env: buildLlamaCppCompatibilityTargetEnv(process.env), + timeoutMs: 84 * 60_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + }, +); function asRecord(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) @@ -112,6 +159,7 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { ], }, }, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + skipOllamaForLlamaCppCompatibility(skip); await artifacts.target.declare({ id: "gpu-e2e", boundary: diff --git a/test/e2e/live/llama-cpp-generic-gpu.test.ts b/test/e2e/live/llama-cpp-generic-gpu.test.ts new file mode 100644 index 00000000000..d90b9980f81 --- /dev/null +++ b/test/e2e/live/llama-cpp-generic-gpu.test.ts @@ -0,0 +1,489 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + MANAGED_LLAMA_CPP_CONTAINER_NAME, + MANAGED_LLAMA_CPP_NETWORK_NAME, +} from "../../../src/lib/inference/llama-cpp/managed-installer.ts"; +import { + loadManagedLlamaCppApiKey, + loadManagedLlamaCppOwner, + loadManagedLlamaCppReceipt, + managedLlamaCppStatePaths, +} from "../../../src/lib/inference/llama-cpp/managed-state.ts"; +import { isLlamaCppServingRecipe } from "../../../src/lib/inference/serving/adapter-registry.ts"; +import { managedInferenceDigest } from "../../../src/lib/inference/serving/catalog-integrity.ts"; +import { loadManagedInferenceCatalog } from "../../../src/lib/inference/serving/catalog-loader.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { + assertAgentExecutionSucceeded, + chatContent, + hasExactReadyPhase, +} from "./gpu-e2e-helpers.ts"; + +const TIMEOUT_MS = 110 * 60_000; +const RECIPE_ID = "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1"; +const PRESET_ID = "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-llamacpp-gpu"; +validateSandboxName(SANDBOX_NAME); + +function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = { + ...buildAvailabilityProbeEnv(process.env), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_LLAMACPP_RECIPE: RECIPE_ID, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROVIDER: "install-llama-cpp", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; + delete selected.NEMOCLAW_MODEL; + return selected; +} + +function loadGenericGpuSetting() { + const catalog = loadManagedInferenceCatalog(); + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === RECIPE_ID); + assert(recipe && isLlamaCppServingRecipe(recipe), "generic GPU E2E llama.cpp recipe is missing"); + const preset = catalog.presets.find(({ metadata }) => metadata.id === PRESET_ID); + assert(preset, "generic GPU E2E preset is missing"); + const modelFile = recipe.spec.model.files[0]; + assert(modelFile && "sizeBytes" in modelFile, "generic GPU E2E GGUF identity is incomplete"); + return { modelFile, preset, recipe }; +} + +test("installs managed llama.cpp on a generic Linux NVIDIA GPU and routes a real agent turn (#8144)", { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "validate exact source and generic NVIDIA GPU host", + "run the declarative managed llama.cpp installer", + "verify exact runtime identity and full GPU offload", + "verify authenticated host and sandbox inference", + "verify OpenClaw agent inference and owned cleanup", + ], + }, +}, async ({ artifacts, cleanup, host, progress, sandbox }) => { + await artifacts.target.declare({ + id: "llama-cpp-generic-gpu", + boundary: + "Linux amd64 host + Docker Engine + one NVIDIA GPU + install.sh managed llama.cpp + OpenShell sandbox route", + configurationAuthority: + "The repository-owned serving recipe and hardware preset supply every model, image, runtime, and serving value.", + credentialBoundary: + "The generated llama.cpp API key remains in owner-only host state and enters commands only through redacted process input.", + }); + + const cleanupEnv = env(); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-gateway", + env: cleanupEnv, + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-sandbox", + env: cleanupEnv, + timeoutMs: 180_000, + }); + + progress.phase("validate exact source and generic NVIDIA GPU host"); + const expectedSha = process.env.NEMOCLAW_E2E_EXPECTED_SHA; + expect(expectedSha, "workflow must bind the exact candidate commit").toMatch(/^[a-f0-9]{40}$/u); + const candidateSha = await host.command("git", ["rev-parse", "HEAD"], { + artifactName: "candidate-commit", + cwd: REPO_ROOT, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(candidateSha.exitCode, resultText(candidateSha)).toBe(0); + expect(candidateSha.stdout.trim()).toBe(expectedSha); + + const architecture = await host.command("uname", ["-m"], { + artifactName: "host-architecture", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(architecture.exitCode, resultText(architecture)).toBe(0); + expect(architecture.stdout.trim()).toBe("x86_64"); + const docker = await host.command("docker", ["info", "--format", "{{json .}}"], { + artifactName: "docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(docker.exitCode, resultText(docker)).toBe(0); + expect(resultText(docker)).not.toMatch(/docker desktop/i); + const nvidia = await host.command( + "nvidia-smi", + ["--query-gpu=name,driver_version,memory.total", "--format=csv,noheader,nounits"], + { + artifactName: "nvidia-smi", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(nvidia.exitCode, resultText(nvidia)).toBe(0); + expect(nvidia.stdout.trim()).toMatch(/^NVIDIA .+,[ ]*[0-9.]+,[ ]*[1-9][0-9]*$/u); + + const { modelFile, preset, recipe } = loadGenericGpuSetting(); + + progress.phase("run the declarative managed llama.cpp installer"); + const install = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: "install-managed-llama-cpp", + cwd: REPO_ROOT, + env: env(), + timeoutMs: 75 * 60_000, + }); + expect(install.exitCode, resultText(install)).toBe(0); + await artifacts.writeText("install-managed-llama-cpp.log", resultText(install)); + + progress.phase("verify exact runtime identity and full GPU offload"); + const paths = managedLlamaCppStatePaths(os.homedir()); + const modelCacheEntry = path.join( + os.homedir(), + ".cache", + "huggingface", + "hub", + `models--${recipe.spec.model.id.replaceAll("/", "--")}`, + "snapshots", + recipe.spec.model.revision, + modelFile.path, + ); + const owner = loadManagedLlamaCppOwner(paths); + const receipt = loadManagedLlamaCppReceipt(paths); + expect(owner).not.toBeNull(); + expect(receipt).not.toBeNull(); + expect(fs.existsSync(modelCacheEntry), "verified GGUF cache entry is missing").toBe(true); + expect(owner).toMatchObject({ + sandboxName: SANDBOX_NAME, + recipeId: RECIPE_ID, + presetDigest: managedInferenceDigest(preset), + recipeDigest: managedInferenceDigest(recipe), + }); + expect(receipt).toMatchObject({ + providerId: "docker", + service: "llama-cpp", + endpoint: { + host: "host.openshell.internal", + networkName: MANAGED_LLAMA_CPP_NETWORK_NAME, + port: recipe.spec.serve.port, + }, + runtime: { + kind: "container", + name: MANAGED_LLAMA_CPP_CONTAINER_NAME, + imageRef: recipe.spec.runtime.image, + model: { + digest: modelFile.digest, + recipeId: RECIPE_ID, + sizeBytes: modelFile.sizeBytes, + }, + }, + }); + + const inspect = await host.command( + "docker", + ["container", "inspect", MANAGED_LLAMA_CPP_CONTAINER_NAME], + { + artifactName: "managed-llama-cpp-container-inspect", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(inspect.exitCode, resultText(inspect)).toBe(0); + const inspectedRuntime = JSON.parse(inspect.stdout) as Array<{ + Config?: { Cmd?: unknown; Image?: unknown }; + HostConfig?: { PortBindings?: Record }; + NetworkSettings?: { Ports?: Record }; + State?: { Pid?: unknown; Running?: unknown }; + }>; + expect(inspectedRuntime).toHaveLength(1); + const runtime = inspectedRuntime[0]; + expect(runtime?.State?.Running).toBe(true); + expect(runtime?.State?.Pid).toEqual(expect.any(Number)); + expect(runtime?.Config?.Image).toBe(recipe.spec.runtime.image); + const containerPid = runtime?.State?.Pid as number; + expect(containerPid).toBeGreaterThan(0); + expect(runtime?.Config?.Cmd).toEqual(expect.any(Array)); + const command = runtime?.Config?.Cmd as string[]; + const gpuLayersIndex = command.indexOf("--gpu-layers"); + expect(gpuLayersIndex).toBeGreaterThanOrEqual(0); + expect(command[gpuLayersIndex + 1]).toBe("all"); + expect(inspectedRuntime[0]?.HostConfig?.PortBindings).toEqual({}); + expect( + Object.values(inspectedRuntime[0]?.NetworkSettings?.Ports ?? {}).every( + (value) => value === null, + ), + ).toBe(true); + const logs = await host.command( + "docker", + ["logs", "--tail", "20000", MANAGED_LLAMA_CPP_CONTAINER_NAME], + { + artifactName: "managed-llama-cpp-container-logs", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(logs.exitCode, resultText(logs)).toBe(0); + const startupLog = resultText(logs); + expect(Buffer.byteLength(startupLog)).toBeGreaterThan(0); + expect(Buffer.byteLength(startupLog)).toBeLessThanOrEqual(16 * 1024 * 1024); + expect(startupLog).not.toMatch( + /no usable GPU|gpu-layers[^\n]*ignored|compiled without[^\n]*GPU|CPU fallback|fallback to CPU|falling back to CPU/iu, + ); + expect(startupLog).toContain("llama_server: model loaded"); + expect(startupLog).toContain( + `llama_server: listening on http://0.0.0.0:${recipe.spec.serve.port}`, + ); + const computeApps = await host.command( + "nvidia-smi", + ["--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader,nounits"], + { + artifactName: "managed-llama-cpp-nvidia-compute-apps", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(computeApps.exitCode, resultText(computeApps)).toBe(0); + const llamaGpuProcess = computeApps.stdout + .trim() + .split("\n") + .map((line) => line.split(",").map((value) => value.trim())) + .find( + ([pid, processName]) => Number(pid) === containerPid && /llama-server$/u.test(processName), + ); + expect(llamaGpuProcess, resultText(computeApps)).toBeDefined(); + const usedGpuMemoryMiB = Number(llamaGpuProcess?.[2]); + const minimumFullOffloadMemoryMiB = Math.floor((modelFile.sizeBytes / 1024 ** 2) * 0.75); + expect(usedGpuMemoryMiB).toBeGreaterThanOrEqual(minimumFullOffloadMemoryMiB); + const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { + artifactName: "managed-llama-cpp-status", + env: env(), + timeoutMs: 120_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + expect(resultText(status)).toContain("Managed llama.cpp: running"); + expect(resultText(status)).toContain(RECIPE_ID); + const doctor = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "doctor"], { + artifactName: "managed-llama-cpp-doctor", + env: env(), + timeoutMs: 120_000, + }); + expect(doctor.exitCode, resultText(doctor)).toBe(0); + + progress.phase("verify authenticated host and sandbox inference"); + const apiKey = loadManagedLlamaCppApiKey(paths); + expect(apiKey, "managed llama.cpp API key is missing").toMatch(/^[a-f0-9]{64}$/u); + artifacts.addRedactionValues([apiKey!]); + const unauthorized = await host.command( + "curl", + [ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + `http://127.0.0.1:${String(recipe.spec.serve.port)}/props`, + ], + { + artifactName: "llama-cpp-unauthorized", + env: env(), + timeoutMs: 30_000, + }, + ); + expect(unauthorized.exitCode, resultText(unauthorized)).toBe(0); + expect(unauthorized.stdout).toBe("401"); + + const health = await host.command( + "curl", + [ + "-fsS", + "-H", + `Authorization: Bearer ${apiKey!}`, + `http://127.0.0.1:${String(recipe.spec.serve.port)}/health`, + ], + { + artifactName: "llama-cpp-health", + env: env(), + redactionValues: [apiKey!], + timeoutMs: 30_000, + }, + ); + expect(health.exitCode, resultText(health)).toBe(0); + expect(health.stdout).toMatch(/ok/i); + + const models = await host.command( + "curl", + [ + "-fsS", + "-H", + `Authorization: Bearer ${apiKey!}`, + `http://127.0.0.1:${String(recipe.spec.serve.port)}/v1/models`, + ], + { + artifactName: "llama-cpp-models", + env: env(), + redactionValues: [apiKey!], + timeoutMs: 30_000, + }, + ); + expect(models.exitCode, resultText(models)).toBe(0); + expect(models.stdout).toContain(recipe.spec.model.servedName); + + const hostChat = await host.command( + "curl", + [ + "-fsS", + "-H", + `Authorization: Bearer ${apiKey!}`, + "-H", + "Content-Type: application/json", + `http://127.0.0.1:${String(recipe.spec.serve.port)}/v1/chat/completions`, + "--data", + JSON.stringify({ + model: recipe.spec.model.servedName, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + max_tokens: 32, + }), + ], + { + artifactName: "llama-cpp-host-chat", + env: env(), + redactionValues: [apiKey!], + timeoutMs: 5 * 60_000, + }, + ); + expect(hostChat.exitCode, resultText(hostChat)).toBe(0); + expect(chatContent(hostChat.stdout)).toMatch(/pong/i); + + const sandboxChat = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `curl -fsS --max-time 300 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${JSON.stringify( + { + model: recipe.spec.model.servedName, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + max_tokens: 32, + }, + )}'`, + ), + { artifactName: "sandbox-inference-local-chat", env: env(), timeoutMs: 6 * 60_000 }, + ); + expect(sandboxChat.exitCode, resultText(sandboxChat)).toBe(0); + expect(chatContent(sandboxChat.stdout)).toMatch(/pong/i); + + progress.phase("verify OpenClaw agent inference and owned cleanup"); + const agent = await host.nemoclaw( + [ + SANDBOX_NAME, + "agent", + "--agent", + "main", + "--json", + "--session-id", + `e2e-llama-cpp-generic-gpu-${Date.now()}-${process.pid}`, + "-m", + "Reply with exactly one word: PONG", + ], + { + artifactName: "openclaw-agent-through-managed-llama-cpp", + env: env(), + timeoutMs: 12 * 60_000, + }, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + assertAgentExecutionSucceeded(agent.stdout, "inference", recipe.spec.model.servedName); + + const readySandbox = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "openshell-sandbox-ready-after-agent", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(readySandbox.exitCode, resultText(readySandbox)).toBe(0); + expect(hasExactReadyPhase(readySandbox.stdout)).toBe(true); + + await artifacts.writeJson("qualification-evidence.json", { + candidateSha: expectedSha, + preset: { id: PRESET_ID, digest: owner!.presetDigest }, + recipe: { id: RECIPE_ID, digest: owner!.recipeDigest }, + model: { + id: recipe.spec.model.id, + digest: modelFile.digest, + servedName: recipe.spec.model.servedName, + }, + runtime: { image: recipe.spec.runtime.image, provider: receipt!.providerId }, + gpu: { + host: nvidia.stdout.trim(), + computeProcess: computeApps.stdout.trim(), + requestedLayers: command[gpuLayersIndex + 1], + usedMemoryMiB: usedGpuMemoryMiB, + minimumFullOffloadMemoryMiB, + }, + probes: { + unauthorizedStatus: 401, + health: "passed", + models: "passed", + status: "passed", + doctor: "passed", + hostChat: "passed", + sandboxChat: "passed", + openClawAgent: "passed", + }, + }); + + const destroy = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "destroy-managed-llama-cpp-sandbox", + env: env(), + timeoutMs: 180_000, + }); + expect(destroy.exitCode, resultText(destroy)).toBe(0); + const runtimeAbsent = await host.command( + "docker", + ["container", "inspect", MANAGED_LLAMA_CPP_CONTAINER_NAME], + { + artifactName: "managed-llama-cpp-container-absent", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(runtimeAbsent.exitCode).toBe(1); + const networkAbsent = await host.command( + "docker", + ["network", "inspect", MANAGED_LLAMA_CPP_NETWORK_NAME], + { + artifactName: "managed-llama-cpp-network-absent", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(networkAbsent.exitCode).toBe(1); + expect(fs.existsSync(paths.stateDir), "destroy must remove managed llama.cpp state").toBe(false); + expect( + fs.existsSync(modelCacheEntry), + "destroy must preserve the shared Hugging Face cache entry", + ).toBe(true); + + await artifacts.target.complete({ + id: "llama-cpp-generic-gpu", + status: "passed", + candidateSha: expectedSha, + fullGpuOffload: true, + model: recipe.spec.model.servedName, + }); +}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index b86375a3f8e..9bdcb8a571a 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -22,6 +22,15 @@ "test/pr-risk-plan.test.ts" ] }, + { + "live": "test/e2e/live/llama-cpp-generic-gpu.test.ts", + "fast": [ + "src/lib/inference/llama-cpp/managed-selection.test.ts", + "src/lib/inference/llama-cpp/managed-installer.test.ts", + "src/lib/onboard/setup-nim-flow.test.ts", + "test/e2e/support/e2e-workflow.test.ts" + ] + }, { "live": "test/e2e/live/managed-image-multiarch-startup.test.ts", "fast": [ diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 8b540efdd79..ebc8bb1416e 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -34,9 +34,11 @@ describe("e2e workflow boundary", () => { ); }); - it("keeps the E2E workflow push-driven, dispatchable, pinned, and artifact-safe", () => { - expect(validateE2eWorkflowBoundary()).toEqual([]); - }); + it( + "keeps the E2E workflow push-driven, dispatchable, pinned, and artifact-safe", + testTimeoutOptions(30_000), + () => expect(validateE2eWorkflowBoundary()).toEqual([]), + ); it("rejects a Launchable environment gate, authorization drift, and secret-guard drift", () => { const workflow = readWorkflow() as { diff --git a/test/e2e/support/gpu-e2e-helpers.test.ts b/test/e2e/support/gpu-e2e-helpers.test.ts index 5a0ae23e122..1321ec0ed08 100644 --- a/test/e2e/support/gpu-e2e-helpers.test.ts +++ b/test/e2e/support/gpu-e2e-helpers.test.ts @@ -10,9 +10,11 @@ import { describe, expect, it } from "vitest"; import { assertAgentExecutionSucceeded, + buildLlamaCppCompatibilityTargetEnv, env, hasExactReadyPhase, openClawModelConfigProjectionScript, + shouldBootstrapLlamaCppGenericGpuTarget, } from "../live/gpu-e2e-helpers.ts"; const GPU_MODEL = "qwen3.5:9b"; @@ -116,6 +118,45 @@ const invalidExecutionProofs: Array<{ ]; describe("GPU E2E helpers", () => { + it("bootstraps the new llama.cpp target through the trusted pre-merge GPU lane", () => { + expect( + shouldBootstrapLlamaCppGenericGpuTarget({ + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }), + ).toBe(true); + }); + + it("keeps the Ollama GPU lane independent once the dedicated llama.cpp lane exists", () => { + expect( + shouldBootstrapLlamaCppGenericGpuTarget({ + E2E_LLAMA_CPP_DEDICATED_LANE: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }), + ).toBe(false); + }); + + it("does not bootstrap the llama.cpp target outside exact-head live E2E", () => { + expect(shouldBootstrapLlamaCppGenericGpuTarget({ NEMOCLAW_RUN_LIVE_E2E: "1" })).toBe(false); + }); + + it("keeps live collection enabled in the sanitized compatibility child", () => { + const childEnv = buildLlamaCppCompatibilityTargetEnv({ + NEMOCLAW_E2E_CORRELATION_ID: "11111111-1111-4111-8111-111111111111", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + NEMOCLAW_E2E_SHARD: "default", + NEMOCLAW_RUN_LIVE_E2E: "1", + UNRELATED_PARENT_VALUE: "must-not-leak", + }); + + expect(childEnv.NEMOCLAW_E2E_CORRELATION_ID).toBe("11111111-1111-4111-8111-111111111111"); + expect(childEnv.NEMOCLAW_E2E_EXPECTED_SHA).toBe("a".repeat(40)); + expect(childEnv.NEMOCLAW_E2E_SHARD).toBe("default"); + expect(childEnv.NEMOCLAW_RUN_LIVE_E2E).toBe("1"); + expect(childEnv.UNRELATED_PARENT_VALUE).toBeUndefined(); + }); + it("forwards the workflow-owned Ollama model pull timeout", () => { expect(env({}, { NEMOCLAW_OLLAMA_PULL_TIMEOUT: "2400" }).NEMOCLAW_OLLAMA_PULL_TIMEOUT).toBe( "2400", diff --git a/test/llama-cpp-dgx-spark-qualification-runner.test.ts b/test/llama-cpp-dgx-spark-qualification-runner.test.ts index dd1736d989a..53b7d535b7e 100644 --- a/test/llama-cpp-dgx-spark-qualification-runner.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-runner.test.ts @@ -337,7 +337,7 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { "--no-agent", ]), ); - expect(valuesAfter(argv, "--publish")).toEqual(["127.0.0.1::8081"]); + expect(valuesAfter(argv, "--publish")).toEqual([]); const agentQualificationArgv = buildServerContainerArgv(testPlan, { apiKeyHostPath: "/work/tmp/api-key", containerName: "qualified-server", @@ -349,7 +349,7 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { runtimeGid: 1001, runtimeUid: 1001, }); - expect(valuesAfter(agentQualificationArgv, "--publish")).toEqual(["127.0.0.1:8081:8081"]); + expect(valuesAfter(agentQualificationArgv, "--publish")).toEqual([]); expect(valuesAfter(argv, "--network")).toEqual(["qualified-internal"]); expect(valuesAfter(argv, "--user")).toEqual(["1001:1001"]); expect(valuesAfter(argv, "--api-key-file")).toEqual(["/run/secrets/llama-cpp-api-key"]); diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index fc0b46d34af..e75ab4274bb 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -169,6 +169,8 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/runtime-provider/current.ts", "src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts", "src/lib/onboard/runtime-provider/docker-llama-cpp-operation.ts", + "src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge-process.ts", + "src/lib/onboard/runtime-provider/docker-llama-cpp-private-bridge.ts", "src/lib/onboard/runtime-provider/docker.ts", "src/lib/onboard/runtime-provider/host-local-create-journal.ts", "src/lib/onboard/runtime-provider/host-local-inference.ts", diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index b64b662b474..6f84612892b 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -40,7 +40,7 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd"; const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 = - "ba3ab24550161d5d229511a0c679be0a3c6442b1ccd3d998d934ad2ca147e678"; + "a6e5816e8033eed53f43a384474f84e51057af131c90c369a1fccc066828419a"; const CLI_ARTIFACT_CONSUMER_JOB_NAMES = [ "agent-turn-latency", "bedrock-runtime-compatible-anthropic", @@ -72,6 +72,7 @@ const CLI_ARTIFACT_CONSUMER_JOB_NAMES = [ "jetson-nvmap-gpu", "kimi-inference-compat", "live", + "llama-cpp-generic-gpu", "mcp-bridge", "mcp-bridge-dev", "messaging-compatible-endpoint", diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 1c5ad149221..8d307700bdf 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1182,6 +1182,41 @@ function validateInferenceRoutingJob(errors: string[], jobs: WorkflowRecord): vo requireRunDoesNotContain(errors, run, "inference-routing-provider-smoke.test.ts"); } +function validateLlamaCppGenericGpuJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "llama-cpp-generic-gpu"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push(`workflow missing ${jobName} job`); + return; + } + if (job["runs-on"] !== "linux-amd64-gpu-rtxpro6000-latest-1") { + errors.push(`${jobName} job must use the reviewed NVIDIA GPU runner`); + } + const jobEnv = asRecord(job.env); + const expectedEnv = { + NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha || github.sha }}", + NEMOCLAW_LLAMACPP_RECIPE: "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1", + NEMOCLAW_PROVIDER: "install-llama-cpp", + NEMOCLAW_SANDBOX_NAME: "e2e-llamacpp-gpu", + }; + for (const [name, expected] of Object.entries(expectedEnv)) { + if (jobEnv[name] !== expected) errors.push(`${jobName} job must set ${name} to ${expected}`); + } + if (Object.hasOwn(jobEnv, "NEMOCLAW_MODEL")) { + errors.push(`${jobName} job must leave NEMOCLAW_MODEL unset so YAML remains authoritative`); + } + if (asRecord(asRecord(jobs["gpu-e2e"]).env).E2E_LLAMA_CPP_DEDICATED_LANE !== "1") { + errors.push("gpu-e2e must disable the pre-merge llama.cpp compatibility bridge"); + } + const run = requireJobStep( + errors, + jobName, + asSteps(job.steps), + "Run generic NVIDIA GPU llama.cpp live test", + ); + requireRunContains(errors, run, "test/e2e/live/llama-cpp-generic-gpu.test.ts"); +} + function jobPassesNvidiaInferenceSecret(job: WorkflowRecord): boolean { return asSteps(job.steps).some( (step) => asRecord(step.env).NVIDIA_INFERENCE_API_KEY !== undefined, @@ -4941,6 +4976,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateFreeStandingJobSelector(errors, jobs, "inference-routing", "inference-routing"); validateInferenceRoutingJob(errors, jobs); validateCloudInferenceJob(errors, jobs); + validateLlamaCppGenericGpuJob(errors, jobs); validateDoubleOnboardJob(errors, jobs); validateHermesE2EJob(errors, jobs); validateHermesTimeoutHeadroom(errors, jobs);