From 514c16e0df96d8c40ca0cddfca7221418a78647c Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Mon, 10 Aug 2026 11:19:46 +0000 Subject: [PATCH 01/10] fix(cli): report inference health from a served request The in-sandbox route probe reads any final HTTP 200-499 as reachable, so a rejected provider credential answered 401 while status reported healthy inference and exited 0. Status and start now send one inference request over the same route and report its result, and start exits nonzero when the request fails. Signed-off-by: Tinson Lai --- .agents/skills/_shared/controlled-words.md | 4 +- docs/deployment/deploy-to-headless-server.mdx | 6 +- .../choose-local-inference-server.mdx | 4 +- docs/inference/set-up-llama-cpp.mdx | 2 +- docs/inference/set-up-ollama.mdx | 3 +- .../set-up-vllm-on-two-dgx-stations.mdx | 4 +- docs/monitoring/monitor-sandbox-activity.mdx | 2 +- docs/reference/commands.mdx | 35 ++++- docs/reference/troubleshooting.mdx | 5 +- docs/security/credential-rotation.mdx | 3 +- ....ts => inference-invocation-probe.test.ts} | 32 +++-- ...light.ts => inference-invocation-probe.ts} | 56 +++++--- .../actions/sandbox/inference-route-health.ts | 131 ++++++++++++++++++ .../sandbox/rebuild-dcode-preflight.ts | 4 +- src/lib/actions/sandbox/start.test.ts | 61 ++++++++ src/lib/actions/sandbox/start.ts | 42 ++++++ .../actions/sandbox/status-inference.test.ts | 10 +- .../status-snapshot-inference-health.test.ts | 81 +++++++++-- .../sandbox/status-snapshot-recovery.test.ts | 1 + src/lib/actions/sandbox/status-snapshot.ts | 75 +++++----- test/cli/helpers.ts | 19 +++ test/cli/sandbox-status-json.test.ts | 41 +++++- test/cli/sandbox-status-text.test.ts | 9 +- test/helpers/rebuild-flow-harness.ts | 4 +- 24 files changed, 520 insertions(+), 114 deletions(-) rename src/lib/actions/sandbox/{rebuild-inference-preflight.test.ts => inference-invocation-probe.test.ts} (78%) rename src/lib/actions/sandbox/{rebuild-inference-preflight.ts => inference-invocation-probe.ts} (61%) diff --git a/.agents/skills/_shared/controlled-words.md b/.agents/skills/_shared/controlled-words.md index 866123359ae..eeac64fe12c 100644 --- a/.agents/skills/_shared/controlled-words.md +++ b/.agents/skills/_shared/controlled-words.md @@ -184,7 +184,7 @@ other literal identifiers. | `custom endpoint` | Technical noun | A user-supplied inference endpoint that is not one of NemoClaw's named provider choices. | compatible endpoint before compatibility is validated | | `hosted inference` | Technical noun | Inference served by a remote provider-operated service. | local inference, cloud model | | `inference` | Technical noun | Model execution that produces a response from an input. | AI, generation when model execution is intended | -| `inference health` | Technical noun | The complete classification from the named `inference.local` `/v1/models` status probe: `reachable` for HTTP `200` through `499`, `unhealthy` for HTTP `500` through `599`, or `unreachable` when no qualifying HTTP response arrives. | model health, successful inference, validation request | +| `inference health` | Technical noun | The classification reported for a sandbox's inference route: the `inference.local` `/v1/models` probe result, and, when that route is reachable and the sandbox records a provider and a model, the result of one inference request sent over the same route. Values are `healthy`, `unauthorized`, `reachable`, `unhealthy`, `unreachable`, and `not probed`. | model health, successful inference | | `inference profile` | Technical noun | A blueprint selection that defines an inference provider type, provider name, endpoint, model, credential input, and route settings. | provider profile, model profile | | `inference request` | Technical noun | One request sent through an inference route to a model. | API call when the inference purpose matters | | `inference route reachability` | Technical noun | The `reachable` inference-health result produced when `https://inference.local/v1/models` returns HTTP `200` through `499`. It does not establish valid credentials, successful model invocation, readiness, compatibility, or support. | inference health, successful inference, validation request | @@ -391,7 +391,7 @@ A result can support more than one claim only when its evidence meets each defin | Class | Claim | Establishes | Does not establish | |---|---|---|---| | Operational | `inference route reachability` | The named `/v1/models` route returned HTTP `200` through `499`. | Valid credentials, successful model invocation, readiness, compatibility, or support. | -| Operational | `inference health` | The named `/v1/models` probe produced a `reachable`, `unhealthy`, or `unreachable` classification. | Valid credentials, successful model invocation, readiness, compatibility, or support. | +| Operational | `inference health` | The named `/v1/models` probe classification, plus the result of one inference request over the same route when NemoClaw sent one. | Broader API conformance, other requests or models, readiness, compatibility, or support. | | Operational | `readiness check` | A service or resource meets named criteria to begin its intended work. | Broader reliability, compatibility, or support. | | Operational | `validation request` | One authenticated request succeeded for the named endpoint, API family, model, and request shape. | Broader API conformance, other requests or models, reliability, or support. | | Evidence | `verification` | Evidence confirms the stated result for the named revision and environment. | Compatibility or support unless the evidence and decision establish them. | diff --git a/docs/deployment/deploy-to-headless-server.mdx b/docs/deployment/deploy-to-headless-server.mdx index f6a229e1ea2..8ab30001168 100644 --- a/docs/deployment/deploy-to-headless-server.mdx +++ b/docs/deployment/deploy-to-headless-server.mdx @@ -235,8 +235,8 @@ $$nemoclaw headless-agent connect --probe-only ``` `$$nemoclaw headless-agent status` exits nonzero when the sandbox, gateway, local container, or authoritative inference route is not verified. -Its main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox. -HTTP `200` through `499` reports `reachable`, while HTTP `500` through `599` reports `unhealthy`. +Its main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox, then sends one inference request over the same route when that probe reports the route reachable. +The line reports `healthy` when the route served the request, `unauthorized` when the route rejected it with HTTP `401` or `403`, and `unhealthy` when the route returned HTTP `500` through `599`. During an SSH session, `status` points to `$$nemoclaw headless-agent dashboard-url` only when the agent gateway is running and loopback dashboard access needs a port forward. @@ -251,7 +251,7 @@ It does not restart or replace the shared host OpenShell gateway. Readiness requires all of these results: - The exact OpenShell sandbox row is `Ready` or `Running`. -- `$$nemoclaw headless-agent status` exits with status `0` and reports the inference route as `reachable`. +- `$$nemoclaw headless-agent status` exits with status `0` and reports the inference route as `healthy`. - `$$nemoclaw headless-agent connect --probe-only` exits with status `0`. ## Access the Dashboard and API diff --git a/docs/inference/choose-local-inference-server.mdx b/docs/inference/choose-local-inference-server.mdx index 359ae3e1e61..8d0fe656f0f 100644 --- a/docs/inference/choose-local-inference-server.mdx +++ b/docs/inference/choose-local-inference-server.mdx @@ -127,8 +127,8 @@ $$nemoclaw my-assistant status $$nemoclaw my-assistant doctor ``` -Accept the result when `status` reports the inference route as `reachable` and `doctor` exits with status `0`. -Route reachability does not by itself establish successful model invocation. +Accept the result when `status` reports the inference route as `healthy` and `doctor` exits with status `0`. +The `healthy` state means the route served one inference request; it does not establish results for other requests or models. Verify the bounded host publication: diff --git a/docs/inference/set-up-llama-cpp.mdx b/docs/inference/set-up-llama-cpp.mdx index fe79833d3e3..b2424bece67 100644 --- a/docs/inference/set-up-llama-cpp.mdx +++ b/docs/inference/set-up-llama-cpp.mdx @@ -155,7 +155,7 @@ $$nemoclaw my-assistant doctor docker port nemoclaw-llama-cpp ``` -Accept the result when `status` reports `Managed llama.cpp: running`, the inference route is `reachable`, and `doctor` exits with status `0`. +Accept the result when `status` reports `Managed llama.cpp: running`, the inference route is `healthy`, and `doctor` exits with status `0`. The Docker port output must contain `127.0.0.1:8081`. These checks do not establish agent and model qualification. diff --git a/docs/inference/set-up-ollama.mdx b/docs/inference/set-up-ollama.mdx index 5952dad3dfd..35bdd7259fa 100644 --- a/docs/inference/set-up-ollama.mdx +++ b/docs/inference/set-up-ollama.mdx @@ -165,7 +165,8 @@ The wizard manages the proxy lifecycle: All proxy endpoints require the token, including `GET /api/tags`. The host-side proxy liveness check treats any HTTP response, including `401`, as evidence that the proxy answered. -The authoritative `status` and `doctor` route probe runs inside the sandbox and reports HTTP `200` through `499` as `healthy`, HTTP `500` through `599` as `unhealthy`, and transport failures or unavailable probes as `unreachable` or `not probed`. +The `status` and `doctor` route probe runs inside the sandbox and reports HTTP `200` through `499` as reachable, HTTP `500` through `599` as `unhealthy`, and transport failures or unavailable probes as `unreachable` or `not probed`. +`status` then sends one inference request over the reachable route and reports `healthy` when the route served it, or `unauthorized` when the proxy token is no longer accepted. If Ollama is already bound to a non-loopback address, onboarding restarts it on `127.0.0.1:11434` so the proxy becomes the only network path. diff --git a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx index 5280e41ebe8..11086b51602 100644 --- a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx +++ b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx @@ -82,8 +82,8 @@ $$nemoclaw my-assistant status ``` Confirm that `inference get` reports the `vllm-local` provider and the `nemotron-ultra` model. -Continue only when the `Inference` row in the status output reports `reachable`. -This result confirms that the sandbox route returned HTTP `200` through `499`; it does not prove a successful model invocation. +Continue only when the `Inference` row in the status output reports `healthy`. +This result confirms that the sandbox route served one inference request; it does not establish results for other requests or models. ## Review Reboot and Resume Behavior diff --git a/docs/monitoring/monitor-sandbox-activity.mdx b/docs/monitoring/monitor-sandbox-activity.mdx index 1f6dfcdbe2d..95caecebdf8 100644 --- a/docs/monitoring/monitor-sandbox-activity.mdx +++ b/docs/monitoring/monitor-sandbox-activity.mdx @@ -122,7 +122,7 @@ If the request fails, check these items. 1. Run `$$nemoclaw status` to confirm the active provider and endpoint. Check the main `Inference` line first. - If it shows `unhealthy`, `unreachable`, or `not probed`, inspect the labeled upstream, local-backend, and auth-proxy diagnostics to identify the failing hop. + If it shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled upstream, local-backend, and auth-proxy diagnostics to identify the failing hop. Restart a local backend only when its own diagnostic fails. 2. Run `$$nemoclaw logs --follow` to view error messages from the blueprint runner. 3. Verify that the host can reach the inference endpoint. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f1956c936ff..515cd4f53dc 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1538,6 +1538,13 @@ Starting an already-running sandbox skips the container start and still runs the A paused container is unpaused. If the container was removed entirely, `start` fails and points you to `$$nemoclaw rebuild`. +After the gateway and forward checks pass, `start` sends one inference request through `https://inference.local` using the sandbox's recorded provider and model. +A gateway that answers the `/v1/models` probe can still reject an inference request, so the command exits non-zero when that request fails. +It prints the probe result, including the HTTP status when the route returned one, and points you to the sandbox doctor command. +Each run sends one 16-token request through the stored provider credential, so `start` waits up to 30 seconds for it and consumes provider tokens on a hosted route. +When the sandbox records no provider or no model, `start` skips the request and exits `0`. +`doctor` still classifies an HTTP `401` or `403` route response as reachable, so correct the provider credential when `start` reports one of those statuses. + ### `$$nemoclaw status` Show sandbox-scoped status, health, and inference configuration for one registered sandbox. @@ -1563,6 +1570,19 @@ When `routeDrift.canConnect` is `false`, `connect` cannot safely restore the rec Refer to [Use Shared Gateway Routes](../inference/manage-inference/use-shared-gateway-routes) for the route-sharing workflow. `openshellDriver` and `openshellVersion` are always strings (falling back to `"unknown"` when the registry has no value), so consumers can rely on `typeof` checks. `failureLayer` is `null` when no preflight failure was detected and otherwise one of `docker_unreachable`, `sandbox_container_stopped`, or `sandbox_dashboard_port_conflict`; when set, `inferenceHealth` is suppressed to `null` so automation does not see a stale remote-provider healthy status during a local outage. +`inferenceHealth.ok` reports whether the inference route served one inference request sent from inside the sandbox, not whether that route answered an HTTP request. +The route probe treats any final HTTP status from `200` through `499` as reachable, so a route with an invalidated provider credential answers HTTP `401` while the route is up. +The request uses the live gateway route's provider and model, and falls back to the recorded values when the live route is unreadable. +Each run sends one 16-token request through the stored provider credential, so `status` waits up to 30 seconds for it and consumes provider tokens on a hosted route. +When NemoClaw sends an inference request, `inferenceHealth.subprobes` reports the route probe result as the `route reachability` hop, so a failing verdict still shows that the route itself answered. +`inferenceHealth.failureLabel` reports why the inference request failed: + +- `unauthorized` when the route rejected it with HTTP `401` or `403`. +- `unhealthy` when the route returned another failing HTTP status. +- `unreachable` when the request returned no HTTP status, including a probe that could not run. + +A host-side upstream probe under `inferenceHealth.subprobes` stays a diagnostic and does not change `inferenceHealth.ok`, because the sandbox route is the one the agent uses. +When the route probe failed, or the sandbox records no provider or no model, NemoClaw skips the inference request and `inferenceHealth` reports the route probe result alone. `dockerPaused` is `true` when NemoClaw detects that the Docker-driver sandbox container is paused. In that case, text output keeps OpenShell's authoritative phase but prints a `docker unpause ` recovery hint instead of sending you directly to rebuild. For terminal runtime sandboxes, the command also checks cgroup OOM kill counters. @@ -1606,19 +1626,22 @@ $$nemoclaw my-assistant status --json $$nemoclaw sandbox status my-assistant --json ``` -The command probes `https://inference.local/v1/models` from inside the sandbox as the authoritative inference health check. -This check exercises the same route that agent traffic uses. +The command probes `https://inference.local/v1/models` from inside the sandbox, and when that probe reports the route reachable it sends one inference request over the same route. +That inference request is the authoritative inference health check, and both checks exercise the route that agent traffic uses. The main `Inference` line reports one of these states: | State | Meaning | |-------|---------| -| `reachable` | The route returned an HTTP status from `200` through `499`. Authentication responses such as `401` and `403` confirm route reachability. | -| `unhealthy` | The route returned an HTTP status from `500` through `599`. | -| `unreachable` | The route had a transport failure, returned no final HTTP status (`000` or an interim `100` through `199`), or returned an invalid status outside `100` through `599`. | +| `healthy` | The route served the inference request. | +| `unauthorized` | The route rejected the inference request with HTTP `401` or `403`. | +| `reachable` | The route returned an HTTP status from `200` through `499` and NemoClaw did not send an inference request. | +| `unhealthy` | The route returned an HTTP status from `500` through `599`, or it rejected the inference request with another failing status. | +| `unreachable` | The route had a transport failure, returned no final HTTP status (`000` or an interim `100` through `199`), returned an invalid status outside `100` through `599`, or the inference request returned no HTTP status. | | `not probed` | NemoClaw could not run the authoritative route probe from a reachable sandbox. | | `not verified` | NemoClaw could not verify the sandbox or gateway state, so it skips inference probing. | -An authentication response confirms that the route is reachable, not that provider credentials are valid. +An authentication response on the route probe alone confirms that the route is reachable, not that provider credentials are valid. +`$$nemoclaw doctor` sends no inference request, so it reports an HTTP `401` or `403` route response as reachable and exits `0` where `status` reports `unauthorized`. The command can also print direct host-side provider checks such as `Inference (upstream)` and provider-specific subprobes. For supported remote providers, this diagnostic sends an authenticated request to the configured model and accepts only a recognized Chat Completions, streaming Chat Completions, or Anthropic Messages response. It uses a 3-second connection timeout, a 5-second total timeout, and an 8-token output limit. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 02e2656e25e..bfeaa2f944b 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1828,8 +1828,9 @@ Check the active provider and endpoint: $$nemoclaw status ``` -The main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox, so it reflects the route the agent actually uses. -If that line shows `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. +The main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox and then sends one inference request over the same route, so it reflects the route the agent uses. +If that line shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. +An `unauthorized` line means the route answered but rejected the request, so refresh the provider credential rather than the route. For local Ollama and local vLLM, `Inference (ollama backend)` or the corresponding local-backend line reports the host-side service separately. For Local Ollama, current releases can also print `Inference (auth proxy)` when a proxy token is available. If a local backend or auth-proxy diagnostic fails, start the backend or re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route. diff --git a/docs/security/credential-rotation.mdx b/docs/security/credential-rotation.mdx index 842f183e475..d1a436f18c2 100644 --- a/docs/security/credential-rotation.mdx +++ b/docs/security/credential-rotation.mdx @@ -197,7 +197,8 @@ Plan for rebuild downtime when automating messaging rotation and recreation down ## Verify the Replacement `$$nemoclaw credentials list` confirms that a provider exists, but it does not reveal or validate the stored value. -`$$nemoclaw status` performs reachability checks without sending cloud API keys, so even an HTTP `401` or `403` can count as reachable. +`$$nemoclaw status` performs host-side reachability checks without sending cloud API keys, so even an HTTP `401` or `403` can count as reachable there. +`$$nemoclaw status` also sends one inference request through the stored provider credential and reports `unauthorized` when the route rejects it with HTTP `401` or `403`. `$$nemoclaw inference get` reports the active route and does not authenticate a model request. Complete a real request through the rotated integration before declaring the rotation successful. diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts b/src/lib/actions/sandbox/inference-invocation-probe.test.ts similarity index 78% rename from src/lib/actions/sandbox/rebuild-inference-preflight.test.ts rename to src/lib/actions/sandbox/inference-invocation-probe.test.ts index 3f267c24f9e..8131c905983 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.test.ts @@ -4,9 +4,9 @@ import { describe, expect, it, vi } from "vitest"; import { - buildRebuildInferenceProbeCommand, - preflightRebuildInferenceRoute, -} from "./rebuild-inference-preflight"; + buildSandboxInferenceInvocationCommand, + probeSandboxInferenceInvocation, +} from "./inference-invocation-probe"; const input = { sandboxName: "dcode-workspace", @@ -15,9 +15,9 @@ const input = { preferredInferenceApi: "openai-completions", }; -describe("atomic rebuild inference preflight", () => { +describe("sandbox inference invocation probe", () => { it("probes the recorded model through inference.local without embedding a credential (#6195)", () => { - const command = buildRebuildInferenceProbeCommand(input); + const command = buildSandboxInferenceInvocationCommand(input); expect(command).toContain("https://inference.local/v1/chat/completions"); expect(command).toContain('"model":"nvidia/nemotron"'); @@ -33,11 +33,12 @@ describe("atomic rebuild inference preflight", () => { stderr: "upstream authentication failed for sk-secret-value-that-is-long-enough", })); - const result = preflightRebuildInferenceRoute(input, { execute }); + const result = probeSandboxInferenceInvocation(input, { execute }); expect(result).toEqual({ ok: false, - detail: "existing sandbox inference probe returned HTTP 401", + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, }); expect(JSON.stringify(result)).not.toContain("sk-secret-value-that-is-long-enough"); }); @@ -49,11 +50,12 @@ describe("atomic rebuild inference preflight", () => { stderr: "upstream echoed canary-replay-marker", })); - const result = preflightRebuildInferenceRoute(input, { execute }); + const result = probeSandboxInferenceInvocation(input, { execute }); expect(result).toEqual({ ok: false, - detail: "existing sandbox inference probe returned HTTP 500", + detail: "sandbox inference invocation probe returned HTTP 500", + httpStatus: 500, }); expect(JSON.stringify(result)).not.toContain("canary-replay-marker"); }); @@ -61,11 +63,11 @@ describe("atomic rebuild inference preflight", () => { it("accepts a successful completion through the stored gateway route (#6195)", () => { const execute = vi.fn(() => ({ status: 0, stdout: "200\n{}", stderr: "" })); - expect(preflightRebuildInferenceRoute(input, { execute })).toEqual({ ok: true }); + expect(probeSandboxInferenceInvocation(input, { execute })).toEqual({ ok: true }); }); it("sends max_completion_tokens for a GPT-5 model on the chat completions route", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "gpt-5.4" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "gpt-5.4" }); expect(command).toContain("https://inference.local/v1/chat/completions"); expect(command).toContain('"max_completion_tokens":16'); @@ -73,21 +75,21 @@ describe("atomic rebuild inference preflight", () => { }); it("sends max_completion_tokens for an o-series model on the chat completions route", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "o3-mini" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "o3-mini" }); expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("keeps max_tokens for a model that supports the legacy chat completions field", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "nvidia/nemotron" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "nvidia/nemotron" }); expect(command).toContain('"max_tokens":16'); expect(command).not.toContain('"max_completion_tokens"'); }); it("sends max_output_tokens on the responses route", () => { - const command = buildRebuildInferenceProbeCommand({ + const command = buildSandboxInferenceInvocationCommand({ ...input, preferredInferenceApi: "openai-responses", }); @@ -106,7 +108,7 @@ describe("atomic rebuild inference preflight", () => { ["anthropic messages", "claude-sonnet-4-6", "anthropic-messages", "max_tokens"], ])("requests a reply budget the endpoint accepts on the %s route (#7939)", (_route, model, preferredInferenceApi, field) => { const endpointMinimumReplyTokens = 16; - const command = buildRebuildInferenceProbeCommand({ + const command = buildSandboxInferenceInvocationCommand({ ...input, model, preferredInferenceApi, diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts similarity index 61% rename from src/lib/actions/sandbox/rebuild-inference-preflight.ts rename to src/lib/actions/sandbox/inference-invocation-probe.ts index 739fb82d13a..aa0c16b6a70 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -6,20 +6,29 @@ import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/m import { shellQuote } from "../../runner"; import { executeSandboxExecCommand, type SandboxCommandResult } from "./process-recovery"; -export type RebuildInferencePreflightInput = { +export type SandboxInferenceInvocationInput = { sandboxName: string; provider: string; model: string; preferredInferenceApi: string | null; }; -export type RebuildInferencePreflightResult = { ok: true } | { ok: false; detail: string }; +export type SandboxInferenceInvocationResult = + | { ok: true } + | { ok: false; detail: string; httpStatus: number | null }; -export type RebuildInferencePreflightDeps = { +export type SandboxInferenceInvocationDeps = { execute?: (sandboxName: string, command: string, timeout?: number) => SandboxCommandResult | null; }; -function buildProbeRequest(input: RebuildInferencePreflightInput): { +/** + * Rebuild preflight recreates the sandbox and tolerates a slow first token. + * Status and start run in an interactive wait and use the shorter timeout. + */ +export const REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS = 100_000; +export const READINESS_INFERENCE_INVOCATION_TIMEOUT_MS = 30_000; + +function buildProbeRequest(input: SandboxInferenceInvocationInput): { endpoint: string; headers: string[]; payload: Record; @@ -63,7 +72,9 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { }; } -export function buildRebuildInferenceProbeCommand(input: RebuildInferencePreflightInput): string { +export function buildSandboxInferenceInvocationCommand( + input: SandboxInferenceInvocationInput, +): string { const request = buildProbeRequest(input); const headerArgs = ["Content-Type: application/json", ...request.headers] .map((header) => `-H ${shellQuote(header)}`) @@ -78,23 +89,36 @@ export function buildRebuildInferenceProbeCommand(input: RebuildInferencePreflig } /** - * Exercise the configured gateway route from the still-running sandbox. The - * request uses OpenShell's stored provider credential through inference.local; - * no host credential is placed in the command or its output. + * Send one minimal agent request over the configured gateway route from the + * still-running sandbox. The request uses OpenShell's stored provider + * credential through inference.local; no host credential is placed in the + * command or its output. */ -export function preflightRebuildInferenceRoute( - input: RebuildInferencePreflightInput, - deps: RebuildInferencePreflightDeps = {}, -): RebuildInferencePreflightResult { +export function probeSandboxInferenceInvocation( + input: SandboxInferenceInvocationInput, + deps: SandboxInferenceInvocationDeps = {}, + timeoutMs: number = REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS, +): SandboxInferenceInvocationResult { const execute = deps.execute ?? executeSandboxExecCommand; - const result = execute(input.sandboxName, buildRebuildInferenceProbeCommand(input), 100_000); + const result = execute( + input.sandboxName, + buildSandboxInferenceInvocationCommand(input), + timeoutMs, + ); if (result?.status === 0) return { ok: true }; - if (!result) return { ok: false, detail: "existing sandbox inference probe was unavailable" }; + if (!result) { + return { + ok: false, + detail: "sandbox inference invocation probe was unavailable", + httpStatus: null, + }; + } const httpStatus = result.stdout.match(/(?:^|\n)([1-5]\d\d)(?:\n|$)/)?.[1]; return { ok: false, detail: httpStatus - ? `existing sandbox inference probe returned HTTP ${httpStatus}` - : `existing sandbox inference probe exited with status ${result.status}`, + ? `sandbox inference invocation probe returned HTTP ${httpStatus}` + : `sandbox inference invocation probe exited with status ${result.status}`, + httpStatus: httpStatus ? Number.parseInt(httpStatus, 10) : null, }; } diff --git a/src/lib/actions/sandbox/inference-route-health.ts b/src/lib/actions/sandbox/inference-route-health.ts index 8a3559bd719..f99d5275577 100644 --- a/src/lib/actions/sandbox/inference-route-health.ts +++ b/src/lib/actions/sandbox/inference-route-health.ts @@ -4,12 +4,22 @@ import { captureOpenshellForStatus, isCommandTimeout } from "../../adapters/openshell/runtime"; import { OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; +import type { ProviderHealthStatus } from "../../inference/health"; import { buildSandboxInferenceRouteProbeArgs, classifyInferenceRouteFailureLabel, isDcodeManagedExecMissingDetail, parseSandboxInferenceRouteProbeResult, } from "./connect-inference-route-probe"; +import { + probeSandboxInferenceInvocation, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + type SandboxInferenceInvocationInput, + type SandboxInferenceInvocationResult, +} from "./inference-invocation-probe"; + +export type { SandboxInferenceInvocationResult } from "./inference-invocation-probe"; +export type ProbeSandboxInferenceInvocation = typeof probeSandboxInferenceInvocation; export type SandboxInferenceRouteHealth = { ok: boolean; @@ -90,3 +100,124 @@ export async function probeSandboxInferenceGatewayHealth( `check the in-sandbox proxy and gateway.`, }; } + +function providerHealthDiagnostics( + providerHealth: ProviderHealthStatus | null, +): ProviderHealthStatus[] { + if (!providerHealth) return []; + const { subprobes = [], ...primary } = providerHealth; + const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" }; + return [labeledPrimary, ...subprobes]; +} + +function classifyInferenceInvocationFailureLabel( + httpStatus: number | null, +): NonNullable { + if (httpStatus === null) return "unreachable"; + if (httpStatus === 401 || httpStatus === 403) return "unauthorized"; + return "unhealthy"; +} + +/** + * Report the reachable route as its own hop so an operator can tell a broken + * route from a reachable route that will not serve an inference request. + */ +function reachableRouteSubprobe( + gateway: SandboxInferenceRouteHealth, + endpoint: string, +): ProviderHealthStatus { + return { + ok: true, + probed: true, + providerLabel: "Inference route", + probeLabel: "route reachability", + endpoint, + detail: gateway.detail, + okLabel: "reachable", + }; +} + +/** + * The route probe reads any final HTTP 200-499 as reachable, so a route with + * an invalidated provider credential answers 401 and still passes it. Health + * therefore reports the result of one inference request, and keeps the route + * probe as a subprobe so a failure shows that the route itself answered. + */ +function buildInvokedRouteHealth( + gateway: SandboxInferenceRouteHealth, + endpoint: string, + invocation: SandboxInferenceInvocationResult, +): ProviderHealthStatus { + if (invocation.ok) { + return { + ok: true, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: "Inference gateway served an inference request on https://inference.local.", + subprobes: [reachableRouteSubprobe(gateway, endpoint)], + }; + } + return { + ok: false, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: `Inference gateway did not serve an inference request: ${invocation.detail}.`, + failureLabel: classifyInferenceInvocationFailureLabel(invocation.httpStatus), + subprobes: [reachableRouteSubprobe(gateway, endpoint)], + }; +} + +export function buildSandboxInferenceRouteHealth( + gateway: SandboxInferenceRouteHealth | null, + providerHealth: ProviderHealthStatus | null, + invocation: SandboxInferenceInvocationResult | null, +): ProviderHealthStatus { + const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models"; + const diagnostics = providerHealthDiagnostics(providerHealth); + let routeHealth: ProviderHealthStatus; + if (gateway?.ok && invocation) { + routeHealth = buildInvokedRouteHealth(gateway, endpoint, invocation); + } else if (gateway) { + routeHealth = { + ok: gateway.ok, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: gateway.detail, + ...(gateway.ok + ? { okLabel: "reachable" } + : { + failureLabel: classifyInferenceRouteFailureLabel(gateway.httpStatus), + }), + }; + } else { + routeHealth = { + ok: false, + probed: false, + providerLabel: "Inference route", + endpoint, + detail: `Could not probe ${endpoint} from inside the sandbox.`, + }; + } + const subprobes = [...(routeHealth.subprobes ?? []), ...diagnostics]; + return subprobes.length > 0 ? { ...routeHealth, subprobes } : routeHealth; +} + +export function runSandboxInferenceInvocationProbe( + input: SandboxInferenceInvocationInput, + probe: ProbeSandboxInferenceInvocation = probeSandboxInferenceInvocation, + onProbeError: (error: unknown) => void = () => {}, +): SandboxInferenceInvocationResult { + try { + return probe(input, {}, READINESS_INFERENCE_INVOCATION_TIMEOUT_MS); + } catch (error) { + onProbeError(error); + return { + ok: false, + detail: "sandbox inference invocation probe could not run", + httpStatus: null, + }; + } +} diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index 67387f43ec5..cfdedaf41ab 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -27,13 +27,13 @@ import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import type { ToolDisclosure } from "../../tool-disclosure"; +import { probeSandboxInferenceInvocation } from "./inference-invocation-probe"; import { DCODE_AGENT_NAME, type ResolvedDcodeRebuildTarget, resolveDcodeRebuildTarget, } from "./rebuild-dcode-target"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; -import { preflightRebuildInferenceRoute } from "./rebuild-inference-preflight"; import { disposePreparedDcodeRebuildImage, type PreparedDcodeRebuildImage, @@ -212,7 +212,7 @@ function requireInferenceRoute( target: ResolvedDcodeRebuildTarget, bail: DcodeRebuildPreflightBail, ): void { - const result = preflightRebuildInferenceRoute({ sandboxName, ...target }); + const result = probeSandboxInferenceInvocation({ sandboxName, ...target }); if (!result.ok) { fail( `recorded inference credentials or route were rejected: ${result.detail}`, diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index e6ebea0d12b..991d675b68b 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -344,4 +344,65 @@ describe("startSandbox", () => { await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("probe exploded"); }); + + it("reports the recorded route as ready only after it serves an agent request", async () => { + const probeInferenceInvocation = vi.fn(() => ({ ok: true }) as const); + const h = harness({ probeInferenceInvocation }); + h.getSandbox.mockReturnValue( + sandbox({ + provider: "ollama-local", + model: "nemotron-3-nano:30b", + preferredInferenceApi: "openai-completions", + }), + ); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(probeInferenceInvocation).toHaveBeenCalledWith( + { + sandboxName: "my-sandbox", + provider: "ollama-local", + model: "nemotron-3-nano:30b", + preferredInferenceApi: "openai-completions", + }, + {}, + 30_000, + ); + expect(probeInferenceInvocation.mock.invocationCallOrder[0]).toBeGreaterThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); + }); + + it("exits nonzero when the started gateway will not serve an agent request", async () => { + const probeInferenceInvocation = vi.fn( + () => + ({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, + }) as const, + ); + const h = harness({ probeInferenceInvocation }); + h.getSandbox.mockReturnValue( + sandbox({ provider: "ollama-local", model: "nemotron-3-nano:30b" }), + ); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(1); + const output = h.log.mock.calls.map(([line]) => line).join("\n"); + expect(output).toContain("HTTP 401"); + expect(output).toContain("doctor"); + }); + + it("stays unattested instead of failing when the sandbox records no route", async () => { + const probeInferenceInvocation = vi.fn(() => ({ ok: true }) as const); + const h = harness({ probeInferenceInvocation }); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(probeInferenceInvocation).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 7e904befd0a..d7812a5acb5 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -7,6 +7,11 @@ import { } from "../../onboard/runtime-provider/access"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +import { + probeSandboxInferenceInvocation, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + type SandboxInferenceInvocationResult, +} from "./inference-invocation-probe"; import { resolveSandboxLifecycleProvider, type SandboxLifecycleResult, @@ -52,9 +57,39 @@ export interface SandboxStartDeps { runtimeProviders?: RuntimeProviderBundleRegistry; restoreStartupState?: (sandboxName: string) => void; verifyGateway?: (sandboxName: string) => Promise; + probeInferenceInvocation?: typeof probeSandboxInferenceInvocation; log?: (message: string) => void; } +/** + * A started gateway that answers the /v1/models probe can still reject an + * inference request, so start sends one inference request with the recorded + * provider and model before it reports success. A registry entry with no + * provider or no model has nothing to request, so start skips the request + * instead of failing. + */ +function checkStartedSandboxInference( + sandboxName: string, + sandbox: SandboxEntry, + deps: SandboxStartDeps, + log: (message: string) => void, +): SandboxInferenceInvocationResult | null { + const model = (sandbox.model ?? "").trim(); + const provider = (sandbox.provider ?? "").trim(); + if (!model || !provider) return null; + log(" Checking that the sandbox serves an agent request…"); + return (deps.probeInferenceInvocation ?? probeSandboxInferenceInvocation)( + { + sandboxName, + provider, + model, + preferredInferenceApi: sandbox.preferredInferenceApi ?? null, + }, + {}, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + ); +} + /** * Restart a stopped sandbox through the lifecycle facet bound to its durable * provider identity, then restore startup state before verifying readiness and @@ -85,6 +120,7 @@ export async function startSandbox( const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; + const readiness: { inference: SandboxInferenceInvocationResult | null } = { inference: null }; await resolved.lifecycle.verifyStarted(input, async (name) => { log(" Restoring sandbox startup state…"); const restoreStartupState = @@ -96,6 +132,12 @@ export async function startSandbox( restoreStartupState(name); log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name); + readiness.inference = checkStartedSandboxInference(name, resolved.sandbox, deps, log); }); + if (readiness.inference && !readiness.inference.ok) { + log(` The sandbox started but inference is not usable: ${readiness.inference.detail}.`); + log(` Run the sandbox doctor command for '${sandboxName}' to identify the failing hop.`); + return { exitCode: 1 }; + } return { exitCode: 0 }; } diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index 3000c418d9f..dc46f8de045 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -57,6 +57,7 @@ describe("sandbox status inference.local route health (#6192)", () => { ? async () => Promise.reject(new Error("openshell unavailable TOKEN=super-secret")) : async () => options.routeHealth, ), + probeSandboxInferenceInvocationImpl: vi.fn(() => ({ ok: true }) as const), probeTerminalRuntimeHealth: vi.fn(() => ({ kind: "ok" as const, oomKillCount: 0 as const })), reportInferenceProbeError, }; @@ -173,9 +174,9 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( expect.objectContaining({ ok: false, probeLabel: "upstream" }), - ]); + ); }); it("probes the live route while status displays the sandbox's recorded route (#6315)", async () => { @@ -219,14 +220,14 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( expect.objectContaining({ ok: false, probed: false, probeLabel: "upstream", detail: "Direct provider health probe could not run.", }), - ]); + ); }); it("preserves local backend and auth-proxy diagnostics beneath the route result", async () => { @@ -261,6 +262,7 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth?.subprobes?.map((probe) => probe.probeLabel)).toEqual([ + "route reachability", "ollama backend", "auth proxy", ]); diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index de7a3eee9c8..6b63206439b 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ProviderHealthStatus } from "../../inference/health"; import type { SandboxEntry } from "../../state/registry"; +import type { SandboxInferenceInvocationResult } from "./inference-invocation-probe"; import type { SandboxInferenceRouteHealth } from "./inference-route-health"; import { collectSandboxStatusSnapshot } from "./status-snapshot"; @@ -16,6 +17,7 @@ import { collectSandboxStatusSnapshot } from "./status-snapshot"; function snapshotDeps( gateway: SandboxInferenceRouteHealth | null, providerHealth: ProviderHealthStatus | null = null, + invocation: SandboxInferenceInvocationResult = { ok: true }, ) { const sandbox: SandboxEntry = { name: "alpha", @@ -38,6 +40,7 @@ function snapshotDeps( }, probeProviderHealthImpl: () => providerHealth, probeSandboxInferenceGatewayHealthImpl: async () => gateway, + probeSandboxInferenceInvocationImpl: () => invocation, }, }; } @@ -129,7 +132,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledTimes(2); expect(delayInferenceRecoveryProbe).toHaveBeenCalledOnce(); expect(delayInferenceRecoveryProbe).toHaveBeenCalledWith(2_000); - expect(snapshot.inferenceHealth).toMatchObject({ ok: true, okLabel: "reachable" }); + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); }); it("reports the inference route as unreachable after all post-recovery probes", async () => { @@ -193,7 +196,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(delayInferenceRecoveryProbe).not.toHaveBeenCalled(); }); - it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => { + it("reports a served agent request as healthy and keeps reachability as its own hop (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, endpoint: "https://inference.local/v1/models", @@ -209,9 +212,20 @@ describe("collectSandboxStatusSnapshot inference route health", () => { probed: true, providerLabel: "Inference route", endpoint: "https://inference.local/v1/models", - okLabel: "reachable", }); expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.okLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.subprobes).toEqual([ + { + ok: true, + probed: true, + providerLabel: "Inference route", + probeLabel: "route reachability", + endpoint: "https://inference.local/v1/models", + detail: gateway.detail, + okLabel: "reachable", + }, + ]); }); it("does not set okLabel for a 5xx route failure, and classifies it unhealthy (#6846)", async () => { @@ -262,7 +276,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined(); }); - it("keeps a failed model-invocation subprobe distinct from the reachable route label (#6846)", async () => { + it("keeps an upstream subprobe failure out of the served-route verdict (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, endpoint: "https://inference.local/v1/models", @@ -284,9 +298,60 @@ describe("collectSandboxStatusSnapshot inference route health", () => { snapshotDeps(gateway, providerHealth), ); - expect(snapshot.inferenceHealth).toMatchObject({ ok: true, okLabel: "reachable" }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ - { ...providerHealth, probeLabel: "upstream" }, - ]); + expect(snapshot.inferenceHealth).toMatchObject({ ok: true }); + expect(snapshot.inferenceHealth?.subprobes).toContainEqual({ + ...providerHealth, + probeLabel: "upstream", + }); + }); + + it("reports an unauthorized verdict when the reachable route rejects an agent request", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 401, + detail: + "Inference gateway responded HTTP 401 on https://inference.local/v1/models (full chain reachable).", + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, null, { + ok: false, + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, + }), + ); + + expect(snapshot.inferenceHealth).toMatchObject({ + ok: false, + probed: true, + failureLabel: "unauthorized", + }); + expect(snapshot.inferenceHealth?.okLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.detail).toContain("HTTP 401"); + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( + expect.objectContaining({ probeLabel: "route reachability", ok: true }), + ); + }); + + it("does not send an agent request when the route probe already failed", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: + "Inference gateway unreachable on https://inference.local/v1/models from inside the sandbox.", + }; + const options = snapshotDeps(gateway); + const probeSandboxInferenceInvocationImpl = vi.fn(() => ({ ok: true }) as const); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, probeSandboxInferenceInvocationImpl }, + }); + + expect(probeSandboxInferenceInvocationImpl).not.toHaveBeenCalled(); + expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unreachable" }); }); }); diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts index 47008ef8e7f..48a1f768c66 100644 --- a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -73,6 +73,7 @@ function snapshotDeps(recoveryResult: unknown) { }, probeProviderHealthImpl, probeSandboxInferenceGatewayHealthImpl, + probeSandboxInferenceInvocationImpl: vi.fn(() => ({ ok: true }) as const), recoverSandboxProcesses: vi.fn(() => recoveryResult) as never, }; } diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index ec0a3dc1b23..8fe4eab9c67 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -35,11 +35,15 @@ import { buildGatewayInferenceGetArgs, canSandboxGatewayRouteRealign, } from "./connect-inference-gateway"; -import { classifyInferenceRouteFailureLabel } from "./connect-inference-route-probe"; import { getSandboxDockerRuntime } from "./docker-health"; import type { SandboxGatewayState } from "./gateway-state"; import { getReconciledSandboxGatewayState, getSandboxGatewayStateForStatus } from "./gateway-state"; -import { probeSandboxInferenceGatewayHealth } from "./inference-route-health"; +import { + buildSandboxInferenceRouteHealth, + type ProbeSandboxInferenceInvocation, + probeSandboxInferenceGatewayHealth, + runSandboxInferenceInvocationProbe, +} from "./inference-route-health"; import { getSandboxStatusPreflight, type SandboxStatusFailureLayer, @@ -103,49 +107,11 @@ export function maybeGetSandboxStatusInferenceHealth( ); } -function providerHealthDiagnostics( - providerHealth: ProviderHealthStatus | null, -): ProviderHealthStatus[] { - if (!providerHealth) return []; - const { subprobes = [], ...primary } = providerHealth; - const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" }; - return [labeledPrimary, ...subprobes]; -} - /** True when the authoritative inference route must make status exit nonzero. */ export function isInferenceHealthFailing(inferenceHealth: ProviderHealthStatus | null): boolean { return Boolean(inferenceHealth && (!inferenceHealth.probed || !inferenceHealth.ok)); } -function buildSandboxInferenceRouteHealth( - gateway: Awaited>, - providerHealth: ProviderHealthStatus | null, -): ProviderHealthStatus { - const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models"; - const diagnostics = providerHealthDiagnostics(providerHealth); - const routeHealth: ProviderHealthStatus = gateway - ? { - ok: gateway.ok, - probed: true, - providerLabel: "Inference route", - endpoint, - detail: gateway.detail, - ...(gateway.ok - ? { okLabel: "reachable" } - : { - failureLabel: classifyInferenceRouteFailureLabel(gateway.httpStatus), - }), - } - : { - ok: false, - probed: false, - providerLabel: "Inference route", - endpoint, - detail: `Could not probe ${endpoint} from inside the sandbox.`, - }; - return diagnostics.length > 0 ? { ...routeHealth, subprobes: diagnostics } : routeHealth; -} - export interface SandboxStatusReport { schemaVersion: 1; name: string; @@ -294,6 +260,7 @@ interface CollectSandboxStatusSnapshotDeps { captureOpenshellForStatusImpl?: typeof captureOpenshellForStatus; probeProviderHealthImpl?: ProbeProviderHealth; probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; + probeSandboxInferenceInvocationImpl?: ProbeSandboxInferenceInvocation; delayInferenceRecoveryProbe?: DelayInferenceRecoveryProbe; reportInferenceProbeError?: (message: string) => void; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; @@ -582,7 +549,33 @@ export async function collectSandboxStatusSnapshot( reportInferenceProbeError(error, opts.deps?.reportInferenceProbeError ?? console.error); gatewayChain = null; } - inferenceHealth = buildSandboxInferenceRouteHealth(gatewayChain, providerHealth); + // Take the provider and model as one pair. Falling back per field can pair + // a live model with a recorded provider and request a route neither one + // describes. + const invocationRoute = + live?.provider && live.model + ? { provider: live.provider, model: live.model } + : { provider: currentProvider, model: currentModel }; + const invocationModel = (invocationRoute.model || "").trim(); + const invocationProvider = (invocationRoute.provider || "").trim(); + const invocation = + gatewayChain?.ok && invocationModel && invocationProvider + ? runSandboxInferenceInvocationProbe( + { + sandboxName, + provider: invocationProvider, + model: invocationModel, + preferredInferenceApi: sb?.preferredInferenceApi ?? null, + }, + opts.deps?.probeSandboxInferenceInvocationImpl, + (error) => + reportInferenceProbeError( + error, + opts.deps?.reportInferenceProbeError ?? console.error, + ), + ) + : null; + inferenceHealth = buildSandboxInferenceRouteHealth(gatewayChain, providerHealth, invocation); } const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); const terminalRuntimeHealth = diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 7ae4ac1a531..45168a067c3 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -8,6 +8,7 @@ import os from "node:os"; import path from "node:path"; import { parse as parseYaml } from "yaml"; +import { SANDBOX_EXEC_STARTED_MARKER } from "../../src/lib/actions/sandbox/sandbox-exec-output"; import type { OwnedTestResources } from "../helpers/owned-test-resources"; import { execTimeout, testTimeout, testTimeoutOptions } from "../helpers/timeouts"; @@ -313,9 +314,27 @@ export function writeHealthyDockerStub(localBin: string): void { ); } +/** + * Answer the agent-request readiness probe inside an `openshell sandbox exec` + * stub. The probe posts a completion over the same transport as the route + * probe, and that transport trusts stdout only after the exec marker. + */ +export function inferenceInvocationStubLines(httpStatus = "200", exitCode = 0): string[] { + return [ + ' case "$*" in', + " *chat/completions*|*/v1/responses*|*/v1/messages*)", + ` printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'`, + ` printf '%s\\n' ${JSON.stringify(httpStatus)}`, + ` exit ${String(exitCode)}`, + " ;;", + " esac", + ]; +} + export function healthyInferenceRouteStubLines(): string[] { return [ 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index 5a2bd94dbf1..435621e497c 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { + inferenceInvocationStubLines, runWithEnv, testTimeoutOptions, writeHealthyDockerStub, @@ -20,6 +21,8 @@ function createInferenceRouteStatusSetup(options: { routeExit?: number; upstreamHttpStatus?: string; upstreamExit?: number; + invocationHttpStatus?: string; + invocationExit?: number; }) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-route-")); const localBin = path.join(home, "bin"); @@ -70,6 +73,7 @@ function createInferenceRouteStatusSetup(options: { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(options.invocationHttpStatus, options.invocationExit), ...(options.executeRouteCommand ? [ ' while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do shift; done', @@ -156,6 +160,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -272,14 +277,42 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { probed: true, endpoint: "https://inference.local/v1/models", }); - expect(parsed.inferenceHealth.subprobes).toEqual([ + expect(parsed.inferenceHealth.subprobes).toContainEqual( expect.objectContaining({ ok: true, probed: false, probeLabel: "upstream" }), - ]); + ); + }); + + it.each([ + 401, 403, + ])("sandbox status --json fails an inference.local HTTP %s that rejects an agent request", (httpStatus) => { + const { home, localBin, sandboxName } = createInferenceRouteStatusSetup({ + routeOutput: `OK ${httpStatus}`, + invocationHttpStatus: String(httpStatus), + invocationExit: 1, + }); + + const result = runWithEnv(`${sandboxName} status --json`, { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(result.code).toBe(1); + const parsed = JSON.parse(result.out); + expect(parsed.inferenceHealth).toMatchObject({ + ok: false, + probed: true, + failureLabel: "unauthorized", + endpoint: "https://inference.local/v1/models", + }); + expect(parsed.inferenceHealth.detail).toContain(String(httpStatus)); + expect(parsed.inferenceHealth.subprobes).toContainEqual( + expect.objectContaining({ ok: true, probeLabel: "route reachability" }), + ); }); it.each([ 401, 403, - ])("sandbox status --json treats an inference.local HTTP %s as healthy (#6192)", (httpStatus) => { + ])("sandbox status --json keeps an inference.local HTTP %s reachable when it still serves an agent request (#6192)", (httpStatus) => { const { home, localBin, sandboxName } = createInferenceRouteStatusSetup({ routeOutput: `OK ${httpStatus}`, }); @@ -334,6 +367,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { [ "#!/usr/bin/env bash", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -649,6 +683,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", diff --git a/test/cli/sandbox-status-text.test.ts b/test/cli/sandbox-status-text.test.ts index 78eb4e24cbc..961aaf5fef9 100644 --- a/test/cli/sandbox-status-text.test.ts +++ b/test/cli/sandbox-status-text.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { healthyInferenceRouteStubLines, + inferenceInvocationStubLines, runWithEnv, testTimeoutOptions, writeHealthyDockerStub, @@ -222,6 +223,7 @@ describe("CLI sandbox status text output", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -260,7 +262,7 @@ describe("CLI sandbox status text output", () => { expect(r.out).toContain("Run `nemoclaw alpha rebuild` to restore."); }); - it("sandbox status reports reachable inference and an unprobed upstream when openshellDriver is not docker", () => { + it("sandbox status reports served inference, its reachability hop, and an unprobed upstream when openshellDriver is not docker", () => { const home = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-non-docker-driver-"), ); @@ -310,7 +312,10 @@ describe("CLI sandbox status text output", () => { expect(r.out).toContain("Sandbox: alpha"); expect(r.out).toContain("Provider: openai-api"); expect(r.out).toContain("Model: gpt-4o-mini"); - expect(r.out).toContain("Inference: reachable (https://inference.local/v1/models)"); + expect(r.out).toContain("Inference: healthy (https://inference.local/v1/models)"); + expect(r.out).toContain( + "Inference (route reachability): reachable (https://inference.local/v1/models)", + ); expect(r.out).toContain("Inference (upstream): not probed"); }); diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index ea64af4382a..40149ef11cb 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -53,7 +53,7 @@ const messagingHostForwardLifecycle = requireDist("./messaging-host-forward-life const messaging = requireDist("../../messaging/index.js"); const mcpBridge = requireDist("./mcp-bridge.js"); const rebuildCustomImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); -const rebuildInference = requireDist("./rebuild-inference-preflight.js"); +const inferenceInvocation = requireDist("./inference-invocation-probe.js"); const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); @@ -529,7 +529,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(nim, "detectGpu").mockReturnValue(null); const routeResults = [...(overrides.dcodeRouteResults ?? [{ ok: true }])]; const preflightDcodeRouteSpy = vi - .spyOn(rebuildInference, "preflightRebuildInferenceRoute") + .spyOn(inferenceInvocation, "probeSandboxInferenceInvocation") .mockImplementation(() => routeResults.shift() ?? { ok: true }); const preparedDcodeBuildContext = { buildCtx: "/tmp/dcode-rebuild-context", From 818e0e1fd0c3f80695d680d542cab8ad3391608a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 11 Aug 2026 11:17:21 -0700 Subject: [PATCH 02/10] fix(status): keep inference route metadata coherent Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 2 +- .../actions/sandbox/status-inference.test.ts | 38 +++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 17 +++++++-- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 51bc8d20f1b..be263d10474 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -40,7 +40,7 @@ "src/lib/actions/sandbox/connect.ts": 38, "src/lib/actions/sandbox/destroy.ts": 29, "src/lib/actions/sandbox/doctor.ts": 30, - "src/lib/actions/sandbox/status-snapshot.ts": 21, + "src/lib/actions/sandbox/status-snapshot.ts": 20, "src/lib/actions/sandbox/policy-channel.ts": 29, "src/lib/actions/sandbox/process-recovery.ts": 21, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index 27647fa2e2a..34be6e301f9 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { buildSandboxInferenceInvocationCommand } from "./inference-invocation-probe"; import { collectSandboxStatusSnapshot, getSandboxStatusInferenceHealth, @@ -15,6 +16,7 @@ describe("sandbox status inference.local route health (#6192)", () => { provider?: string; liveProvider?: string; liveModel?: string; + preferredInferenceApi?: string; providerHealth?: ReturnType; providerProbeThrows?: boolean; routeHealth: { @@ -32,6 +34,7 @@ describe("sandbox status inference.local route health (#6192)", () => { agent: options.agent ?? "openclaw", model: "nvidia/nemotron", provider, + preferredInferenceApi: options.preferredInferenceApi, }; return { getSandbox: () => sandbox, @@ -206,6 +209,41 @@ describe("sandbox status inference.local route health (#6192)", () => { }); }); + it("does not apply the recorded API family to a different live route", async () => { + const deps = snapshotDeps({ + provider: "compatible-endpoint", + preferredInferenceApi: "openai-responses", + liveProvider: "openai-api", + liveModel: "gpt-5.2", + routeHealth: { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "route reachable", + }, + }); + deps.probeSandboxInferenceInvocationImpl.mockImplementation((input) => { + const command = buildSandboxInferenceInvocationCommand(input); + expect(command).toContain("https://inference.local/v1/chat/completions"); + expect(command).not.toContain("https://inference.local/v1/responses"); + return { ok: true }; + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); + expect(deps.probeSandboxInferenceInvocationImpl).toHaveBeenCalledWith( + { + sandboxName: "alpha", + provider: "openai-api", + model: "gpt-5.2", + preferredInferenceApi: null, + }, + {}, + 30_000, + ); + }); + it("keeps inference.local authoritative when the upstream diagnostic throws (#6192)", async () => { const deps = snapshotDeps({ providerProbeThrows: true, diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index d6287cda600..534472967f5 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -566,8 +566,19 @@ export async function collectSandboxStatusSnapshot( // describes. const invocationRoute = live?.provider && live.model - ? { provider: live.provider, model: live.model } - : { provider: currentProvider, model: currentModel }; + ? { + provider: live.provider, + model: live.model, + // The live gateway RPC does not expose a stored API override. Do + // not combine its provider/model with the recorded sandbox's API + // family; resolve the live route's compatible default instead. + preferredInferenceApi: null, + } + : { + provider: currentProvider, + model: currentModel, + preferredInferenceApi: sb?.preferredInferenceApi ?? null, + }; const invocationModel = (invocationRoute.model || "").trim(); const invocationProvider = (invocationRoute.provider || "").trim(); const invocation = @@ -577,7 +588,7 @@ export async function collectSandboxStatusSnapshot( sandboxName, provider: invocationProvider, model: invocationModel, - preferredInferenceApi: sb?.preferredInferenceApi ?? null, + preferredInferenceApi: invocationRoute.preferredInferenceApi, }, opts.deps?.probeSandboxInferenceInvocationImpl, (error) => From a4b22d891967b956b3882833acd8e95f181bcb3d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 11 Aug 2026 11:28:53 -0700 Subject: [PATCH 03/10] test(status): type the route probe mock Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/status-inference.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index 34be6e301f9..3e850ab53fe 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -60,7 +60,10 @@ describe("sandbox status inference.local route health (#6192)", () => { ? async () => Promise.reject(new Error("openshell unavailable TOKEN=super-secret")) : async () => options.routeHealth, ), - probeSandboxInferenceInvocationImpl: vi.fn(() => ({ ok: true }) as const), + probeSandboxInferenceInvocationImpl: vi.fn( + (_input: Parameters[0]) => + ({ ok: true }) as const, + ), probeTerminalRuntimeHealth: vi.fn(() => ({ kind: "ok" as const, oomKillCount: 0 as const })), reportInferenceProbeError, }; From 0c6b9a6ad5799068667c5cfe8552190d51294838 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Tue, 11 Aug 2026 11:47:01 -0700 Subject: [PATCH 04/10] test(inference): update rebuild probe harness Signed-off-by: Apurv Kumaria --- test/helpers/rebuild-flow-dcode-harness.ts | 2 +- test/helpers/rebuild-flow-harness.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/helpers/rebuild-flow-dcode-harness.ts b/test/helpers/rebuild-flow-dcode-harness.ts index c5f0aefba55..dd32e501f68 100644 --- a/test/helpers/rebuild-flow-dcode-harness.ts +++ b/test/helpers/rebuild-flow-dcode-harness.ts @@ -413,7 +413,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(nim, "detectGpu").mockReturnValue(null); const routeResults = [...(overrides.dcodeRouteResults ?? [{ ok: true }])]; const preflightDcodeRouteSpy = vi - .spyOn(rebuildInference, "preflightRebuildInferenceRoute") + .spyOn(rebuildInference, "probeSandboxInferenceInvocation") .mockImplementation(() => routeResults.shift() ?? { ok: true }); const preparedDcodeBuildContext = { buildCtx: "/tmp/dcode-rebuild-context", diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 177383f6842..e7f84a49a8e 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -50,7 +50,7 @@ export const processRecovery = requireDist("./process-recovery.js"); export const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependencies.js"); export const rebuildCustomImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); export const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); -export const rebuildInference = requireDist("./rebuild-inference-preflight.js"); +export const rebuildInference = requireDist("./inference-invocation-probe.js"); export const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); export const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); export const rebuildPreparedImageContext = requireDist("./rebuild-prepared-image-context.js"); From 282a38b7acb13e728f432c5e674db0ab09d9222d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Tue, 11 Aug 2026 12:10:12 -0700 Subject: [PATCH 05/10] test(status): serve invocation in recovery fixture Signed-off-by: Apurv Kumaria --- test/nemoclaw-cli-recovery.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/nemoclaw-cli-recovery.test.ts b/test/nemoclaw-cli-recovery.test.ts index e0bccfb09ed..139ea8c00c9 100644 --- a/test/nemoclaw-cli-recovery.test.ts +++ b/test/nemoclaw-cli-recovery.test.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, it } from "vitest"; +import { SANDBOX_EXEC_STARTED_MARKER } from "../src/lib/actions/sandbox/sandbox-exec-output"; import { testTimeoutOptions } from "./helpers/timeouts"; describe("nemoclaw CLI runtime recovery", () => { @@ -84,6 +85,11 @@ if (args[0] === "sandbox" && args[1] === "get" && (args[2] === "my-assistant" || } if (args[0] === "sandbox" && args[1] === "exec") { + const command = args.join(" "); + if (command.includes("chat/completions") || command.includes("/v1/responses") || command.includes("/v1/messages")) { + process.stdout.write(${JSON.stringify(`${SANDBOX_EXEC_STARTED_MARKER}\n200\n`)}); + process.exit(0); + } process.stdout.write("OK 200\\n"); process.exit(0); } From 2112136f156fb653f895ff890135c7deed49a0f0 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 12 Aug 2026 10:52:09 -0700 Subject: [PATCH 06/10] fix(status): preserve API family for aligned route Signed-off-by: Apurv Kumaria --- .../actions/sandbox/status-inference.test.ts | 35 +++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 7 ++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index 3e850ab53fe..e173724474d 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -247,6 +247,41 @@ describe("sandbox status inference.local route health (#6192)", () => { ); }); + it("preserves the recorded Responses API for an unchanged live route (#8731)", async () => { + const deps = snapshotDeps({ + provider: "compatible-endpoint", + preferredInferenceApi: "openai-responses", + liveProvider: "compatible-endpoint", + liveModel: "nvidia/nemotron", + routeHealth: { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "route reachable", + }, + }); + deps.probeSandboxInferenceInvocationImpl.mockImplementation((input) => { + const command = buildSandboxInferenceInvocationCommand(input); + expect(command).toContain("https://inference.local/v1/responses"); + expect(command).not.toContain("https://inference.local/v1/chat/completions"); + return { ok: true }; + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); + expect(deps.probeSandboxInferenceInvocationImpl).toHaveBeenCalledWith( + { + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "nvidia/nemotron", + preferredInferenceApi: "openai-responses", + }, + {}, + 30_000, + ); + }); + it("keeps inference.local authoritative when the upstream diagnostic throws (#6192)", async () => { const deps = snapshotDeps({ providerProbeThrows: true, diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 3bf055e3a88..301cf7a5103 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -602,9 +602,10 @@ export async function collectSandboxStatusSnapshot( provider: live.provider, model: live.model, // The live gateway RPC does not expose a stored API override. Do - // not combine its provider/model with the recorded sandbox's API - // family; resolve the live route's compatible default instead. - preferredInferenceApi: null, + // not carry an API family across route drift. When the live pair + // is unchanged, the recorded family still describes that route. + preferredInferenceApi: + routeDriftPlan?.kind === "aligned" ? (sb?.preferredInferenceApi ?? null) : null, } : { provider: currentProvider, From d50658ee3f1f4d4ab7a7e44f161b794fc6bfa108 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 12 Aug 2026 10:59:52 -0700 Subject: [PATCH 07/10] docs(inference): align status route health guidance Signed-off-by: Apurv Kumaria --- docs/inference/verify-inference-route.mdx | 9 ++++++--- docs/reference/commands.mdx | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 8c9f0efccbc..6c06af63d7a 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -30,9 +30,12 @@ Run the named sandbox status command. $$nemoclaw status ``` -The `Inference` row checks the sandbox's `inference.local` path and reports it as `reachable` when the route returns HTTP `200` through `499`. -This includes authentication responses such as HTTP `401` and `403`: they confirm route access, not valid provider credentials or a successful model invocation. -Use the status command's upstream provider diagnostic or send a short agent request to verify the configured model. +The `Inference` row first checks the sandbox's `inference.local` path. +When that route responds, `status` sends one inference request through the same path. +The row reports `healthy` only when the route serves that request. +An HTTP `401` or `403` response reports `unauthorized`. +Correct the stored provider credential. +The route-reachability and upstream provider subprobes remain available to identify the failing hop. The provider, model, and endpoint appear with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. When onboarding prints a dashboard summary, use it to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 4cf7fbcc41b..d8d23b2a3a3 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1653,6 +1653,8 @@ Refer to [Use Shared Gateway Routes](../inference/manage-inference/use-shared-ga `inferenceHealth.ok` reports whether the inference route served one inference request sent from inside the sandbox, not whether that route answered an HTTP request. The route probe treats any final HTTP status from `200` through `499` as reachable, so a route with an invalidated provider credential answers HTTP `401` while the route is up. The request uses the live gateway route's provider and model, and falls back to the recorded values when the live route is unreadable. +When the live provider and model match the recorded route, the request also uses the sandbox's recorded API family, including `openai-responses`. +During route drift, NemoClaw does not carry the sandbox's recorded API family to the live provider and model. Each run sends one 16-token request through the stored provider credential, so `status` waits up to 30 seconds for it and consumes provider tokens on a hosted route. When NemoClaw sends an inference request, `inferenceHealth.subprobes` reports the route probe result as the `route reachability` hop, so a failing verdict still shows that the route itself answered. `inferenceHealth.failureLabel` reports why the inference request failed: From d64b86e46fe9949f9d532fee20e5c06308369ae8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 12 Aug 2026 11:18:13 -0700 Subject: [PATCH 08/10] test(docs): align inference health contract Signed-off-by: Apurv Kumaria --- test/inference-options-docs.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 0a453be5640..b0611f53519 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -530,7 +530,7 @@ describe("inference setup navigation", () => { expect(endpoint).toContain("Port `8000` is one of NemoClaw's bundled host-gateway ports."); expect(vllm).toContain("Docker's `--restart unless-stopped` policy"); expect(verification).toContain( - "The `Inference` row checks the sandbox's `inference.local` path", + "The `Inference` row first checks the sandbox's `inference.local` path", ); }); }); From e0df8084833a2b63b700a746fd930e85bf899cf5 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 12 Aug 2026 13:29:23 -0700 Subject: [PATCH 09/10] fix(inference): validate readiness response bodies Signed-off-by: Apurv Kumaria --- docs/inference/verify-inference-route.mdx | 4 +- docs/reference/commands.mdx | 13 ++-- .../inference-invocation-probe.test.ts | 65 ++++++++++++++++++- .../sandbox/inference-invocation-probe.ts | 29 ++++++++- src/lib/inference/health.ts | 44 +++++++++++-- 5 files changed, 141 insertions(+), 14 deletions(-) diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 6c06af63d7a..13a194b9344 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -32,7 +32,9 @@ $$nemoclaw status The `Inference` row first checks the sandbox's `inference.local` path. When that route responds, `status` sends one inference request through the same path. -The row reports `healthy` only when the route serves that request. +The row reports `healthy` only when the route returns a structurally valid result for the recorded API family. +An empty body, malformed JSON, provider-error envelope, or wrong response shape reports `unhealthy`, even with a 2xx status. +Status diagnostics do not include the response body. An HTTP `401` or `403` response reports `unauthorized`. Correct the stored provider credential. The route-reachability and upstream provider subprobes remain available to identify the failing hop. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ce9c5926c03..9b8a6cd3ee5 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1634,7 +1634,7 @@ A paused container is unpaused. If the container was removed entirely, `start` fails and points you to `$$nemoclaw rebuild`. After the gateway and forward checks pass, `start` sends one inference request through `https://inference.local` using the sandbox's recorded provider and model. -A gateway that answers the `/v1/models` probe can still reject an inference request, so the command exits non-zero when that request fails. +A gateway that answers the `/v1/models` probe can still reject an inference request or return an invalid result, so the command exits non-zero in either case. It prints the probe result, including the HTTP status when the route returned one, and points you to the sandbox doctor command. Each run sends one 16-token request through the stored provider credential, so `start` waits up to 30 seconds for it and consumes provider tokens on a hosted route. When the sandbox records no provider or no model, `start` skips the request and exits `0`. @@ -1666,7 +1666,10 @@ Refer to [Use Shared Gateway Routes](../inference/manage-inference/use-shared-ga `openshellDriver` and `openshellVersion` are always strings (falling back to `"unknown"` when the registry has no value), so consumers can rely on `typeof` checks. `agent` is always a string and reports `openclaw` when the registry records no agent for the sandbox. `failureLayer` is `null` when no preflight failure was detected and otherwise one of `docker_unreachable`, `sandbox_container_stopped`, or `sandbox_dashboard_port_conflict`; when set, `inferenceHealth` is suppressed to `null` so automation does not see a stale remote-provider healthy status during a local outage. -`inferenceHealth.ok` reports whether the inference route served one inference request sent from inside the sandbox, not whether that route answered an HTTP request. +`inferenceHealth.ok` reports whether the inference route returned a structurally valid result for one request sent from inside the sandbox. +The result must match Chat Completions, Responses, or Anthropic Messages for the selected route. +An empty body, malformed JSON, provider-error envelope, or wrong response shape reports `unhealthy`, even with a 2xx status. +The probe captures at most 64 KiB and does not include the response body in diagnostics. The route probe treats any final HTTP status from `200` through `499` as reachable, so a route with an invalidated provider credential answers HTTP `401` while the route is up. The request uses the live gateway route's provider and model, and falls back to the recorded values when the live route is unreadable. When the live provider and model match the recorded route, the request also uses the sandbox's recorded API family, including `openai-responses`. @@ -1676,7 +1679,7 @@ When NemoClaw sends an inference request, `inferenceHealth.subprobes` reports th `inferenceHealth.failureLabel` reports why the inference request failed: - `unauthorized` when the route rejected it with HTTP `401` or `403`. -- `unhealthy` when the route returned another failing HTTP status. +- `unhealthy` when the route returned another failing HTTP status or an invalid 2xx response body. - `unreachable` when the request returned no HTTP status, including a probe that could not run. A host-side upstream probe under `inferenceHealth.subprobes` stays a diagnostic and does not change `inferenceHealth.ok`, because the sandbox route is the one the agent uses. @@ -1730,10 +1733,10 @@ The main `Inference` line reports one of these states: | State | Meaning | |-------|---------| -| `healthy` | The route served the inference request. | +| `healthy` | The route returned a structurally valid result for the inference request. | | `unauthorized` | The route rejected the inference request with HTTP `401` or `403`. | | `reachable` | The route returned an HTTP status from `200` through `499` and NemoClaw did not send an inference request. | -| `unhealthy` | The route returned an HTTP status from `500` through `599`, or it rejected the inference request with another failing status. | +| `unhealthy` | The route returned HTTP `500` through `599`, another failing status, or an invalid 2xx response body. | | `unreachable` | The route had a transport failure, returned no final HTTP status (`000` or an interim `100` through `199`), returned an invalid status outside `100` through `599`, or the inference request returned no HTTP status. | | `not probed` | NemoClaw could not run the authoritative route probe from a reachable sandbox. | | `not verified` | NemoClaw could not verify the sandbox or gateway state, so it skips inference probing. | diff --git a/src/lib/actions/sandbox/inference-invocation-probe.test.ts b/src/lib/actions/sandbox/inference-invocation-probe.test.ts index 8131c905983..84d649f1a61 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.test.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.test.ts @@ -24,6 +24,10 @@ describe("sandbox inference invocation probe", () => { expect(command).not.toMatch(/api[_-]?key|authorization|bearer/i); expect(command).not.toMatch(/curl\s+[^;]*-[^-\s]*k/); expect(command).not.toContain("head -c"); + expect(command).toContain("umask 077"); + expect(command).toContain("mktemp /tmp/nemoclaw-inference-invocation.XXXXXX"); + expect(command).toContain("--max-filesize 65536"); + expect(command).not.toContain("-o /dev/null"); }); it("fails closed and redacts diagnostics when the stored gateway credential is rejected (#6195)", () => { @@ -61,11 +65,70 @@ describe("sandbox inference invocation probe", () => { }); it("accepts a successful completion through the stored gateway route (#6195)", () => { - const execute = vi.fn(() => ({ status: 0, stdout: "200\n{}", stderr: "" })); + const execute = vi.fn(() => ({ + status: 0, + stdout: '200\n{"choices":[{"message":{"content":"OK"}}]}', + stderr: "", + })); expect(probeSandboxInferenceInvocation(input, { execute })).toEqual({ ok: true }); }); + it.each([ + ["openai-completions", '{"choices":[{"message":{"content":"OK"}}]}'], + [ + "openai-responses", + '{"output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}', + ], + ["anthropic-messages", '{"content":[{"type":"text","text":"OK"}]}'], + ])("accepts a valid %s response body", (preferredInferenceApi, body) => { + const execute = vi.fn(() => ({ status: 0, stdout: `200\n${body}`, stderr: "" })); + + expect( + probeSandboxInferenceInvocation({ ...input, preferredInferenceApi }, { execute }), + ).toEqual({ ok: true }); + }); + + it.each([ + ["Chat Completions", "openai-completions", "an empty response", "204\n"], + ["Chat Completions", "openai-completions", "malformed JSON", "200\nnot-json"], + [ + "Chat Completions", + "openai-completions", + "an error envelope", + '200\n{"error":{"message":"provider failed"}}', + ], + ["Chat Completions", "openai-completions", "the wrong result shape", '200\n{"choices":[]}'], + ["Responses", "openai-responses", "an empty response", "204\n"], + ["Responses", "openai-responses", "malformed JSON", "200\nnot-json"], + [ + "Responses", + "openai-responses", + "an error envelope", + '200\n{"error":{"message":"provider failed"}}', + ], + ["Responses", "openai-responses", "the wrong result shape", '200\n{"output":[]}'], + ["Anthropic Messages", "anthropic-messages", "an empty response", "204\n"], + ["Anthropic Messages", "anthropic-messages", "malformed JSON", "200\nnot-json"], + [ + "Anthropic Messages", + "anthropic-messages", + "an error envelope", + '200\n{"error":{"message":"provider failed"}}', + ], + ["Anthropic Messages", "anthropic-messages", "the wrong result shape", '200\n{"content":[]}'], + ])("rejects %s %s", (_api, preferredInferenceApi, _case, stdout) => { + const execute = vi.fn(() => ({ status: 0, stdout, stderr: "" })); + + expect( + probeSandboxInferenceInvocation({ ...input, preferredInferenceApi }, { execute }), + ).toEqual({ + ok: false, + detail: "sandbox inference invocation probe returned an invalid response body", + httpStatus: Number.parseInt(stdout.slice(0, 3), 10), + }); + }); + it("sends max_completion_tokens for a GPT-5 model on the chat completions route", () => { const command = buildSandboxInferenceInvocationCommand({ ...input, model: "gpt-5.4" }); diff --git a/src/lib/actions/sandbox/inference-invocation-probe.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts index aa0c16b6a70..d7501a1470f 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { getSandboxInferenceConfig } from "../../inference/config"; +import { validateInferenceResponseBody } from "../../inference/health"; import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field"; import { shellQuote } from "../../runner"; import { executeSandboxExecCommand, type SandboxCommandResult } from "./process-recovery"; @@ -27,6 +28,7 @@ export type SandboxInferenceInvocationDeps = { */ export const REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS = 100_000; export const READINESS_INFERENCE_INVOCATION_TIMEOUT_MS = 30_000; +const INFERENCE_INVOCATION_MAX_RESPONSE_BYTES = 64 * 1024; function buildProbeRequest(input: SandboxInferenceInvocationInput): { endpoint: string; @@ -82,9 +84,12 @@ export function buildSandboxInferenceInvocationCommand( const payload = shellQuote(JSON.stringify(request.payload)); const endpoint = shellQuote(request.endpoint); return [ - `code=$(curl -sS --connect-timeout 5 --max-time 90 -o /dev/null -w '%{http_code}' ${headerArgs} --data-binary ${payload} ${endpoint}) || { rc=$?; printf 'curl-error:%s\\n' "$rc"; exit "$rc"; }`, + "umask 077", + "body=$(mktemp /tmp/nemoclaw-inference-invocation.XXXXXX) || exit 1", + "trap 'rm -f \"$body\"' EXIT HUP INT TERM", + `code=$(curl -sS --connect-timeout 5 --max-time 90 --max-filesize ${INFERENCE_INVOCATION_MAX_RESPONSE_BYTES} -o "$body" -w '%{http_code}' ${headerArgs} --data-binary ${payload} ${endpoint}) || { rc=$?; printf 'curl-error:%s\\n' "$rc"; exit "$rc"; }`, "printf '%s\\n' \"$code\"", - 'case "$code" in 2??) exit 0 ;; *) exit 1 ;; esac', + 'case "$code" in 2??) cat "$body"; exit 0 ;; *) exit 1 ;; esac', ].join("; "); } @@ -105,7 +110,6 @@ export function probeSandboxInferenceInvocation( buildSandboxInferenceInvocationCommand(input), timeoutMs, ); - if (result?.status === 0) return { ok: true }; if (!result) { return { ok: false, @@ -113,6 +117,25 @@ export function probeSandboxInferenceInvocation( httpStatus: null, }; } + if (result.status === 0) { + const separator = result.stdout.indexOf("\n"); + const statusText = (separator >= 0 ? result.stdout.slice(0, separator) : result.stdout).trim(); + const httpStatus = /^2\d\d$/.test(statusText) ? Number.parseInt(statusText, 10) : null; + const body = separator >= 0 ? result.stdout.slice(separator + 1) : ""; + const inferenceApi = getSandboxInferenceConfig( + input.model, + input.provider, + input.preferredInferenceApi, + ).inferenceApi; + if (httpStatus !== null && validateInferenceResponseBody(inferenceApi, body).ok) { + return { ok: true }; + } + return { + ok: false, + detail: "sandbox inference invocation probe returned an invalid response body", + httpStatus, + }; + } const httpStatus = result.stdout.match(/(?:^|\n)([1-5]\d\d)(?:\n|$)/)?.[1]; return { ok: false, diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 65ee0df86e5..8fb730a07c0 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -166,7 +166,7 @@ function buildAnthropicMessagesProbeCurlArgs( ]; } -type ResponseValidation = { ok: true } | { ok: false; reason: string }; +export type InferenceResponseValidation = { ok: true } | { ok: false; reason: string }; function isJsonRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); @@ -272,7 +272,7 @@ function isValidAnthropicContentBlock(value: unknown): boolean { return false; } -function validateChatCompletionsResponse(body: string): ResponseValidation { +function validateChatCompletionsResponse(body: string): InferenceResponseValidation { const parsed = parseJsonRecord(body); if (parsed) { if (hasProviderErrorEnvelope(parsed)) { @@ -304,7 +304,32 @@ function validateChatCompletionsResponse(body: string): ResponseValidation { : { ok: false, reason: "response was not a Chat Completions result" }; } -function validateAnthropicMessagesResponse(body: string): ResponseValidation { +function isValidResponsesContentBlock(value: unknown): boolean { + if (!isJsonRecord(value) || typeof value.type !== "string") return false; + if (value.type === "output_text") return typeof value.text === "string"; + if (value.type === "refusal") return typeof value.refusal === "string"; + return false; +} + +function validateResponsesResponse(body: string): InferenceResponseValidation { + const parsed = parseJsonRecord(body); + if (!parsed) return { ok: false, reason: "response was not a Responses result" }; + if (hasProviderErrorEnvelope(parsed)) { + return { ok: false, reason: "provider returned an error envelope" }; + } + if (!Array.isArray(parsed.output) || parsed.output.length === 0) { + return { ok: false, reason: "response was not a Responses result" }; + } + const hasMessage = parsed.output.some((item) => { + if (!isJsonRecord(item) || item.type !== "message" || !Array.isArray(item.content)) { + return false; + } + return item.content.length > 0 && item.content.every(isValidResponsesContentBlock); + }); + return hasMessage ? { ok: true } : { ok: false, reason: "response was not a Responses result" }; +} + +function validateAnthropicMessagesResponse(body: string): InferenceResponseValidation { const parsed = parseJsonRecord(body); if (!parsed) return { ok: false, reason: "response was not an Anthropic Messages result" }; if (hasProviderErrorEnvelope(parsed)) { @@ -319,9 +344,20 @@ function validateAnthropicMessagesResponse(body: string): ResponseValidation { : { ok: false, reason: "response was not an Anthropic Messages result" }; } +export function validateInferenceResponseBody( + inferenceApi: string, + body: string, +): InferenceResponseValidation { + if (inferenceApi === "anthropic-messages") return validateAnthropicMessagesResponse(body); + if (inferenceApi === "openai-responses" || inferenceApi === "responses") { + return validateResponsesResponse(body); + } + return validateChatCompletionsResponse(body); +} + function validateInvocationProbeResult( result: CurlProbeResult, - validateResponse: (body: string) => ResponseValidation, + validateResponse: (body: string) => InferenceResponseValidation, ): CurlProbeResult { if (!result.ok) return result; const validation = validateResponse(result.body); From a36cc957db0efea8d3eae3f15eefbf1780483efb Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 12 Aug 2026 14:04:09 -0700 Subject: [PATCH 10/10] test(inference): align readiness response fixtures Signed-off-by: Apurv Kumaria --- test/cli/helpers.ts | 22 ++++++++++++++++++++++ test/cli/status-gateway-lifecycle.test.ts | 9 ++++++++- test/nemoclaw-cli-recovery.test.ts | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 45168a067c3..e05dbc34515 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -320,11 +320,33 @@ export function writeHealthyDockerStub(localBin: string): void { * probe, and that transport trusts stdout only after the exec marker. */ export function inferenceInvocationStubLines(httpStatus = "200", exitCode = 0): string[] { + const bodyLines = + new Map([ + [ + 0, + [ + ' case "$*" in', + ` *chat/completions*) printf '%s\\n' ${JSON.stringify( + JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), + )} ;;`, + ` */v1/responses*) printf '%s\\n' ${JSON.stringify( + JSON.stringify({ + output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }], + }), + )} ;;`, + ` */v1/messages*) printf '%s\\n' ${JSON.stringify( + JSON.stringify({ content: [{ type: "text", text: "OK" }] }), + )} ;;`, + " esac", + ], + ], + ]).get(exitCode) ?? []; return [ ' case "$*" in', " *chat/completions*|*/v1/responses*|*/v1/messages*)", ` printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'`, ` printf '%s\\n' ${JSON.stringify(httpStatus)}`, + ...bodyLines, ` exit ${String(exitCode)}`, " ;;", " esac", diff --git a/test/cli/status-gateway-lifecycle.test.ts b/test/cli/status-gateway-lifecycle.test.ts index 0b183226e36..476fe66e953 100644 --- a/test/cli/status-gateway-lifecycle.test.ts +++ b/test/cli/status-gateway-lifecycle.test.ts @@ -6,7 +6,13 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { execTimeout, runWithEnv, testTimeout, writeSandboxRegistry } from "./helpers"; +import { + execTimeout, + inferenceInvocationStubLines, + runWithEnv, + testTimeout, + writeSandboxRegistry, +} from "./helpers"; describe("CLI status gateway lifecycle process contracts", () => { it( @@ -98,6 +104,7 @@ describe("CLI status gateway lifecycle process contracts", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', + ...inferenceInvocationStubLines(), ' case "$*" in', " *inference.local/v1/models*) echo 'OK 200' ;;", " *) echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'; echo 'RUNNING' ;;", diff --git a/test/nemoclaw-cli-recovery.test.ts b/test/nemoclaw-cli-recovery.test.ts index 139ea8c00c9..cdd43c79b12 100644 --- a/test/nemoclaw-cli-recovery.test.ts +++ b/test/nemoclaw-cli-recovery.test.ts @@ -87,7 +87,7 @@ if (args[0] === "sandbox" && args[1] === "get" && (args[2] === "my-assistant" || if (args[0] === "sandbox" && args[1] === "exec") { const command = args.join(" "); if (command.includes("chat/completions") || command.includes("/v1/responses") || command.includes("/v1/messages")) { - process.stdout.write(${JSON.stringify(`${SANDBOX_EXEC_STARTED_MARKER}\n200\n`)}); + process.stdout.write(${JSON.stringify(`${SANDBOX_EXEC_STARTED_MARKER}\n200\n{"choices":[{"message":{"role":"assistant","content":"OK"}}]}\n`)}); process.exit(0); } process.stdout.write("OK 200\\n");