diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index b75ae797de..de7a3eee9c 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -84,16 +84,113 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(order).toEqual(["reconcile", "recover-agent-and-forward", "probe-inference"]); }); + it("waits for the inference route after recovering the agent gateway", async () => { + const unreachable: SandboxInferenceRouteHealth = { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: "unreachable", + }; + const healthy: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "reachable", + }; + const options = snapshotDeps(unreachable); + options.deps.reconcile = async () => ({ + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }); + const probeSandboxInferenceGatewayHealthImpl = vi + .fn() + .mockResolvedValueOnce(unreachable) + .mockResolvedValueOnce(healthy); + const delayInferenceRecoveryProbe = vi.fn(async () => undefined); + const recoverSandboxProcesses = vi.fn(() => ({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + })); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { + ...options.deps, + delayInferenceRecoveryProbe, + probeSandboxInferenceGatewayHealthImpl, + recoverSandboxProcesses, + }, + }); + + expect(probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledTimes(2); + expect(delayInferenceRecoveryProbe).toHaveBeenCalledOnce(); + expect(delayInferenceRecoveryProbe).toHaveBeenCalledWith(2_000); + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, okLabel: "reachable" }); + }); + + it("reports the inference route as unreachable after all post-recovery probes", async () => { + const unreachable: SandboxInferenceRouteHealth = { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: "unreachable", + }; + const options = snapshotDeps(unreachable); + options.deps.reconcile = async () => ({ + state: "present", + output: "Phase: Ready", + recoveredSandbox: true, + recoverySandboxVia: "started-stopped-original", + }); + const probeSandboxInferenceGatewayHealthImpl = vi.fn(async () => unreachable); + const delayInferenceRecoveryProbe = vi.fn(async () => undefined); + const recoverSandboxProcesses = vi.fn(() => ({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + })); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { + ...options.deps, + delayInferenceRecoveryProbe, + probeSandboxInferenceGatewayHealthImpl, + recoverSandboxProcesses, + }, + }); + + expect(probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledTimes(3); + expect(delayInferenceRecoveryProbe).toHaveBeenCalledTimes(2); + expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unreachable" }); + }); + it("does not mutate the agent or host forward during an ordinary present status lookup", async () => { const options = snapshotDeps(null); const recoverSandboxProcesses = vi.fn(); + const delayInferenceRecoveryProbe = vi.fn(async () => undefined); + const probeSandboxInferenceGatewayHealthImpl = vi.fn( + options.deps.probeSandboxInferenceGatewayHealthImpl, + ); await collectSandboxStatusSnapshot("alpha", { ...options, - deps: { ...options.deps, recoverSandboxProcesses }, + deps: { + ...options.deps, + delayInferenceRecoveryProbe, + probeSandboxInferenceGatewayHealthImpl, + recoverSandboxProcesses, + }, }); expect(recoverSandboxProcesses).not.toHaveBeenCalled(); + expect(probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledOnce(); + expect(delayInferenceRecoveryProbe).not.toHaveBeenCalled(); }); it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => { diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 636910f863..ec0a3dc1b2 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { setTimeout as sleep } from "node:timers/promises"; import { detectOpenShellStateRpcResultIssue, type OpenShellStateRpcIssue, @@ -55,6 +56,10 @@ type ProbeProviderHealth = ( options?: ProviderHealthProbeOptions, ) => ProviderHealthStatus | null; type ProbeSandboxInferenceGatewayHealth = typeof probeSandboxInferenceGatewayHealth; +type DelayInferenceRecoveryProbe = (delayMs: number) => Promise; + +const RECOVERED_INFERENCE_PROBE_ATTEMPTS = 3; +const RECOVERED_INFERENCE_PROBE_DELAY_MS = 2_000; /** * Honest serving-process state while the self-report response and probe @@ -289,6 +294,7 @@ interface CollectSandboxStatusSnapshotDeps { captureOpenshellForStatusImpl?: typeof captureOpenshellForStatus; probeProviderHealthImpl?: ProbeProviderHealth; probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; + delayInferenceRecoveryProbe?: DelayInferenceRecoveryProbe; reportInferenceProbeError?: (message: string) => void; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; recoverSandboxProcesses?: RecoverSandboxProcesses; @@ -422,6 +428,7 @@ export async function collectSandboxStatusSnapshot( (sb.agent ?? "openclaw") === "openclaw" && parseSandboxPhase(lookup.output || "") === "Ready" && !opts.preflight?.failure; + let recoveredManagedGateway = false; if ( lookup.state === "present" && (lookup.recoveredSandbox || managedOpenClawDeliveryMustBeProven) @@ -438,6 +445,8 @@ export async function collectSandboxStatusSnapshot( }, ); failure = processRecoveryFailure(recovery); + recoveredManagedGateway = + failure === null && recovery.wasRunning === false && recovery.recovered === true; } catch (error) { failure = { layer: "recovery-error", @@ -559,9 +568,14 @@ export async function collectSandboxStatusSnapshot( if (!suppressInferenceProbe && lookup.state === "present") { let gatewayChain: Awaited> = null; try { - gatewayChain = await ( - opts.deps?.probeSandboxInferenceGatewayHealthImpl ?? probeSandboxInferenceGatewayHealth - )(sandboxName); + const probe = + opts.deps?.probeSandboxInferenceGatewayHealthImpl ?? probeSandboxInferenceGatewayHealth; + const attempts = recoveredManagedGateway ? RECOVERED_INFERENCE_PROBE_ATTEMPTS : 1; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + gatewayChain = await probe(sandboxName); + if (gatewayChain?.ok || attempt === attempts) break; + await (opts.deps?.delayInferenceRecoveryProbe ?? sleep)(RECOVERED_INFERENCE_PROBE_DELAY_MS); + } } catch (error) { // This is a permanent fail-closed runtime boundary, but unexpected // OpenShell/transport exceptions must remain observable for diagnosis. diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 324dd80b3f..ee9f7f7020 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -874,7 +874,7 @@ describe("createSetupNim", () => { expect(handleVllmSelection).toHaveBeenCalledOnce(); expect(prompt).not.toHaveBeenCalled(); expect(result).toMatchObject({ provider: "vllm" }); - }); + }, 15_000); it("reuses an already-running local vLLM on a DGX Spark non-interactive run with no requested provider (#7293)", async () => { const profile = { name: "DGX Spark" } as VllmProfile; diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index ae286d6c5e..41c635a729 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -971,7 +971,7 @@ const releasePath = process.argv[3]; ); await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 200 })), ).rejects.toThrow("Sandbox mutation containment is active"); expect(fs.existsSync(reaperPath)).toBe(true); expect(fs.existsSync(containmentPath)).toBe(true);