diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6ce087c432e..616588d2419 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2068,9 +2068,19 @@ $$nemoclaw onboard ``` The variable accepts seconds and applies to the readiness wait only. -When the deadline expires, NemoClaw tries to delete the partially created sandbox. +When the ordinary create deadline expires, NemoClaw tries to delete the partially created sandbox. After successful cleanup, the output ends with `Retry: $$nemoclaw onboard`. If cleanup fails, NemoClaw instead reports that the failed sandbox could not be removed and prints `Manual cleanup: openshell sandbox delete ""`. + +The failure path differs when NemoClaw recreates an OpenShell-managed Docker runtime immediately before this wait. +NemoClaw pins the exact OpenShell sandbox ID before recreation. +Within the same deadline, NemoClaw requires two consecutive `Ready` observations that each confirm the exact ID and successful command execution. +It retries only OpenShell's exact `sandbox is not ready` response. +If the deadline expires, the ID changes, or another probe fails, NemoClaw preserves diagnostics and attempts to restore the pre-recreation Docker container. +If restoration fails, NemoClaw reports that the sandbox and container state is uncertain. +NemoClaw does not start dashboard or other host forwarding, and it does not delete a sandbox by its mutable name. +It leaves the sandbox in place for inspection and recovery. + If readiness still fails after the extended budget, inspect the gateway and sandbox status: ```bash diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index eebacbacd3d..d4841e0200f 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -51,7 +51,15 @@ export function createGpuFlowInput(): SandboxGpuCreateFlowInput { export function createGpuFlowDeps(): SandboxGpuCreateFlowDeps { return { - runOpenshell: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn((args: string[]) => + args[0] === "sandbox" && args[1] === "get" + ? { + status: 0, + stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + stderr: "", + } + : { status: 0, stdout: "", stderr: "" }, + ), runCaptureOpenshell: vi.fn(() => "alpha Ready"), sleep: vi.fn(), openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]), diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f640f20ce7f..c090f8f3520 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -111,6 +111,24 @@ const DEFAULT_RUNTIME_SNAPSHOT = { containerId: "container-a", }; +type OpenShellResult = ReturnType; + +function readySandboxGetResult(): OpenShellResult { + return { + status: 0, + stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + stderr: "", + }; +} + +function createSequencedOpenShellRunner( + entries: Array<[string, OpenShellResult[]]>, +): SandboxGpuCreateFlowDeps["runOpenshell"] { + const resultsByCommand = new Map(entries); + return (args) => + resultsByCommand.get(args.join(" "))?.shift() ?? { status: 0, stdout: "", stderr: "" }; +} + function failNativeCreate(output = "error: unexpected argument '--gpu' found"): void { mocks.streamSandboxCreate.mockResolvedValueOnce({ status: 1, output, sawProgress: false }); } @@ -563,6 +581,125 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled(); expect(createHandoff).toEqual(["poll", "create-complete", "ensure-applied"]); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledWith( + expect.objectContaining({ + stableReadyPolls: 2, + checkReadyIdentity: expect.any(Function), + }), + ); + }); + + it("does not delete a recreated sandbox when the exact readiness probe fails (#9050)", async () => { + const input = createInput(); + const patch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValueOnce(patch); + input.sandboxGpuConfig = { + ...input.sandboxGpuConfig, + mode: "0", + sandboxGpuEnabled: false, + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create"]; + input.persistStartupCommand = true; + input.requiredUlimits = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ]; + const deps = createDeps(); + vi.mocked(deps.runOpenshell).mockImplementation( + createSequencedOpenShellRunner([ + ["sandbox get alpha", [readySandboxGetResult(), readySandboxGetResult()]], + [ + "sandbox exec --name alpha -- true", + [{ status: 1, stdout: "", stderr: "permission denied" }], + ], + ]), + ); + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce((options) => { + expect(options.checkReadyIdentity?.()).toBe("probe_failed"); + return { + ready: false, + reason: "identity_probe_failed", + failurePhase: null, + }; + }); + mockExit(); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); + + expect(patch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { + backupPath: null, + }); + expect(errorOutput()).toContain( + "NemoClaw left the sandbox in place for inspection and recovery", + ); + }); + + it("keeps a transient recreated-sandbox not-ready response inside the readiness wait (#9050)", async () => { + const input = createInput(); + const patch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValueOnce(patch); + input.sandboxGpuConfig = { + ...input.sandboxGpuConfig, + mode: "0", + sandboxGpuEnabled: false, + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create"]; + input.persistStartupCommand = true; + input.requiredUlimits = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, + ]; + const deps = createDeps(); + vi.mocked(deps.runOpenshell).mockImplementation( + createSequencedOpenShellRunner([ + [ + "sandbox get alpha", + [readySandboxGetResult(), readySandboxGetResult(), readySandboxGetResult()], + ], + [ + "sandbox exec --name alpha -- true", + [ + { + status: 1, + stdout: "", + stderr: + `Error: × code: 'The system is not in a state required for the operation's\n` + + ' │ execution\', message: "sandbox is not ready"\n', + }, + { status: 0, stdout: "", stderr: "" }, + ], + ], + ]), + ); + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce((options) => { + expect(options.checkReadyIdentity?.()).toBe("not_ready"); + expect(options.checkReadyIdentity?.()).toBe("ready"); + return { ready: true, reason: "ready", failurePhase: null }; + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "none", + }); + + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.filter(([args]) => args.join(" ") === "sandbox exec --name alpha -- true"), + ).toHaveLength(2); + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); }); it("does not replace a native GPU container solely to persist its startup command", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 5b91caa7368..629a11dd364 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity"; +import { + parseOpenShellSandboxId, + resolveOpenShellSandboxId, +} from "../adapters/openshell/sandbox-identity"; import { printSandboxCreateRecoveryHints } from "../build-context"; import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; @@ -27,6 +30,7 @@ import type { SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; +import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; import { addTraceEvent } from "./tracing"; @@ -44,6 +48,61 @@ export type SandboxGpuCreateAttemptState = { // to live validation or the GPU proof. const REPLACEMENT_STABLE_READY_POLLS = 2; +const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/gu; +const OPENSHELL_SANDBOX_NOT_READY = + /^Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"$/iu; + +type OpenShellCommandResult = ReturnType; + +function normalizedOpenShellCommandOutput(result: OpenShellCommandResult): string { + return `${String(result.stderr ?? "")}\n${String(result.stdout ?? "")}` + .replace(ANSI_RE, "") + .replace(/[×│]/gu, " ") + .replace(/\s+/gu, " ") + .trim(); +} + +type OpenShellSandboxIdentityProbe = + | { state: "identified"; sandboxId: string } + | { state: "not_ready" } + | { state: "failed" }; + +function probeExactOpenShellSandboxId( + sandboxName: string, + deps: SandboxGpuCreateFlowDeps, +): OpenShellSandboxIdentityProbe { + const result = deps.runOpenshell(["sandbox", "get", sandboxName], { + ignoreError: true, + suppressOutput: true, + }); + if (result.status === 0 && !result.error) { + const sandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); + return sandboxId ? { state: "identified", sandboxId } : { state: "failed" }; + } + return OPENSHELL_SANDBOX_NOT_READY.test(normalizedOpenShellCommandOutput(result)) + ? { state: "not_ready" } + : { state: "failed" }; +} + +function checkRecreatedSandboxReadyIdentity( + sandboxName: string, + expectedSandboxId: string, + deps: SandboxGpuCreateFlowDeps, +): ReturnType { + const identity = probeExactOpenShellSandboxId(sandboxName, deps); + if (identity.state === "not_ready") return "not_ready"; + if (identity.state === "failed") return "probe_failed"; + if (identity.sandboxId !== expectedSandboxId) return "identity_changed"; + const result = deps.runOpenshell(["sandbox", "exec", "--name", sandboxName, "--", "true"], { + ignoreError: true, + suppressOutput: true, + }); + if (result.status === 0 && !result.error) return "ready"; + return OPENSHELL_SANDBOX_NOT_READY.test(normalizedOpenShellCommandOutput(result)) + ? "not_ready" + : "probe_failed"; +} + class ManagedBootstrapCreateStreamFailure extends Error { constructor(readonly result: Awaited>) { super("Managed bootstrap held workload did not complete its create stream."); @@ -334,6 +393,21 @@ export function createSandboxGpuCreateAttemptRunner( ); } } + const preRecreateIdentity = deferRestartSafeCutover + ? probeExactOpenShellSandboxId(input.sandboxName, deps) + : null; + const expectedRecreatedSandboxId = + preRecreateIdentity?.state === "identified" ? preRecreateIdentity.sandboxId : null; + if (deferRestartSafeCutover && !expectedRecreatedSandboxId) { + console.error(""); + console.error( + ` Sandbox '${input.sandboxName}' reached Ready, but OpenShell did not return one exact durable sandbox ID before runtime recreation.`, + ); + printSandboxCreateFailureDiagnostics(input.sandboxName, { + backupPath: input.restoreBackupPath, + }); + process.exit(createResult.status === 0 ? 1 : createResult.status); + } await runtimePatch.ensureApplied(); await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); @@ -343,7 +417,14 @@ export function createSandboxGpuCreateAttemptRunner( runCaptureOpenshell: deps.runCaptureOpenshell, isSandboxReady, getSandboxFailurePhase, - stableReadyPolls: compatibility || managedBootstrap ? REPLACEMENT_STABLE_READY_POLLS : 1, + stableReadyPolls: + compatibility || managedBootstrap || expectedRecreatedSandboxId + ? REPLACEMENT_STABLE_READY_POLLS + : 1, + checkReadyIdentity: expectedRecreatedSandboxId + ? () => + checkRecreatedSandboxReadyIdentity(input.sandboxName, expectedRecreatedSandboxId, deps) + : undefined, sleep: deps.sleep, }); if (!readiness.ready) { @@ -388,7 +469,11 @@ export function createSandboxGpuCreateAttemptRunner( backupPath: input.restoreBackupPath, }); if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); - else { + else if (expectedRecreatedSandboxId) { + console.error( + " NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for inspection and recovery.", + ); + } else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, suppressOutput: true, diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 452ef12bc62..0965dd2748a 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -86,6 +86,64 @@ describe("createSandboxReadyWaiter", () => { }); describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => { + it("waits for the exact recreated sandbox to become executable before accepting stable Ready (#9050)", () => { + const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready`]); + const checkReadyIdentity = vi.fn().mockReturnValueOnce("not_ready").mockReturnValue("ready"); + + expect( + waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + timeoutSecs: 30, + runCaptureOpenshell, + isSandboxReady, + stableReadyPolls: 2, + checkReadyIdentity, + sleep, + }), + ).toEqual({ ready: true, reason: "ready", failurePhase: null }); + expect(checkReadyIdentity).toHaveBeenCalledTimes(3); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it("stops when the recreated sandbox identity changes (#9050)", () => { + const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready`]); + + expect( + waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + timeoutSecs: 30, + runCaptureOpenshell, + isSandboxReady, + checkReadyIdentity: () => "identity_changed", + sleep, + }), + ).toEqual({ ready: false, reason: "identity_changed", failurePhase: null }); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("stops after an unknown recreated-runtime probe failure (#9050)", () => { + const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready`]); + + expect( + waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + timeoutSecs: 30, + runCaptureOpenshell, + isSandboxReady, + checkReadyIdentity: () => "probe_failed", + sleep, + }), + ).toEqual({ + ready: false, + reason: "identity_probe_failed", + failurePhase: null, + }); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(sleep).not.toHaveBeenCalled(); + }); + it("does not probe when the readiness deadline is zero (#3768)", () => { const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready`]); diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 763b3da375a..8358fd3b4ce 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -81,8 +81,16 @@ export function getSandboxReadyErrorDebouncePolls( export type CreatedSandboxReadinessResult = | { ready: true; reason: "ready"; failurePhase: null } | { ready: false; reason: "terminal_failure_phase"; failurePhase: string | null } + | { ready: false; reason: "identity_changed"; failurePhase: null } + | { ready: false; reason: "identity_probe_failed"; failurePhase: null } | { ready: false; reason: "timeout"; failurePhase: null }; +export type CreatedSandboxReadyIdentityCheck = () => + | "ready" + | "not_ready" + | "identity_changed" + | "probe_failed"; + export interface SandboxReadyWaitDeps { runCaptureOpenshell: RunCaptureOpenshell; isSandboxReady: (output: string, sandboxName: string) => boolean; @@ -204,6 +212,13 @@ export function waitForCreatedSandboxReadyWithTrace(options: { * from reaching the GPU proof. */ stableReadyPolls?: number; + /** + * Optional exact-identity and executability proof for a sandbox whose + * runtime was replaced after OpenShell first reported Ready. A transient + * not-ready result stays inside this bounded wait. Identity changes and + * all other probe failures remain terminal. + */ + checkReadyIdentity?: CreatedSandboxReadyIdentityCheck; /** * Consecutive Error-phase polls required before the wait treats the phase as * terminal. Defaults to {@link getSandboxReadyErrorDebouncePolls} (30 polls). @@ -267,6 +282,32 @@ export function waitForCreatedSandboxReadyWithTrace(options: { attempt += 1; const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); if (isSandboxReady(list, sandboxName)) { + const identity = options.checkReadyIdentity?.() ?? "ready"; + if (identity === "identity_changed") { + addTraceEvent("identity_changed", { attempt }); + result = { + ready: false, + reason: "identity_changed", + failurePhase: null, + }; + return true; + } + if (identity === "probe_failed") { + addTraceEvent("identity_probe_failed", { attempt }); + result = { + ready: false, + reason: "identity_probe_failed", + failurePhase: null, + }; + return true; + } + if (identity === "not_ready") { + consecutiveReadyPolls = 0; + consecutiveFailurePolls = 0; + lastFailurePhase = null; + addTraceEvent("ready_identity_pending", { attempt }); + return false; + } consecutiveReadyPolls += 1; consecutiveFailurePolls = 0; lastFailurePhase = null; @@ -362,6 +403,12 @@ export function formatCreatedSandboxReadinessFailureMessage( const phase = readiness.failurePhase ?? "a terminal failure"; return ` Sandbox '${sandboxName}' entered ${phase} phase before it became ready (waited up to ${timeoutSecs}s).`; } + if (readiness.reason === "identity_changed") { + return ` Sandbox '${sandboxName}' changed identity before its recreated runtime became ready.`; + } + if (readiness.reason === "identity_probe_failed") { + return ` NemoClaw could not verify that sandbox '${sandboxName}' still had the expected ID and accepted commands.`; + } return ` Sandbox '${sandboxName}' was created but did not become ready within ${timeoutSecs}s.`; }