fix(onboard): gate forwarding on recreated readiness

docs/reference/troubleshooting.mdx

@@ -2068,9 +2068,19 @@ $$nemoclaw onboard
 ```
 
 The variable accepts seconds and applies to the readiness wait only.
-When the deadline expires, NemoClaw tries to delete the partially created sandbox.
+When the ordinary create deadline expires, NemoClaw tries to delete the partially created sandbox.
 After successful cleanup, the output ends with `Retry: $$nemoclaw onboard`.
 If cleanup fails, NemoClaw instead reports that the failed sandbox could not be removed and prints `Manual cleanup: openshell sandbox delete "<name>"`.
+
+The failure path differs when NemoClaw recreates an OpenShell-managed Docker runtime immediately before this wait.
+NemoClaw pins the exact OpenShell sandbox ID before recreation.
+Within the same deadline, NemoClaw requires two consecutive `Ready` observations that each confirm the exact ID and successful command execution.
+It retries only OpenShell's exact `sandbox is not ready` response.
+If the deadline expires, the ID changes, or another probe fails, NemoClaw preserves diagnostics and attempts to restore the pre-recreation Docker container.
+If restoration fails, NemoClaw reports that the sandbox and container state is uncertain.
+NemoClaw does not start dashboard or other host forwarding, and it does not delete a sandbox by its mutable name.
+It leaves the sandbox in place for inspection and recovery.
+
 If readiness still fails after the extended budget, inspect the gateway and sandbox status:
 
 ```bash

src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts

@@ -51,7 +51,15 @@ export function createGpuFlowInput(): SandboxGpuCreateFlowInput {
 
 export function createGpuFlowDeps(): SandboxGpuCreateFlowDeps {
   return {
-    runOpenshell: vi.fn(() => ({ status: 0 })),
+    runOpenshell: vi.fn((args: string[]) =>
+      args[0] === "sandbox" && args[1] === "get"
+        ? {
+            status: 0,
+            stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n",
+            stderr: "",
+          }
+        : { status: 0, stdout: "", stderr: "" },
+    ),
     runCaptureOpenshell: vi.fn(() => "alpha Ready"),
     sleep: vi.fn(),
     openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]),

src/lib/onboard/sandbox-gpu-create-flow.test.ts

@@ -563,6 +563,127 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => {
     );
     expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled();
     expect(createHandoff).toEqual(["poll", "create-complete", "ensure-applied"]);
+    expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledWith(
+      expect.objectContaining({
+        stableReadyPolls: 2,
+        checkReadyIdentity: expect.any(Function),
+      }),
+    );
+  });
+
+  it("does not delete a recreated sandbox when the exact readiness probe fails (#9050)", async () => {
+    const input = createInput();
+    const patch = createPatch();
+    mocks.createDockerGpuSandboxCreatePatch.mockReturnValueOnce(patch);
+    input.sandboxGpuConfig = {
+      ...input.sandboxGpuConfig,
+      mode: "0",
+      sandboxGpuEnabled: false,
+    };
+    input.gpuRoutePlan = "none";
+    input.initialGpuRoute = "none";
+    input.createArgv = ["openshell", "sandbox", "create"];
+    input.persistStartupCommand = true;
+    input.requiredUlimits = [
+      { name: "nproc", soft: 512, hard: 512 },
+      { name: "nofile", soft: 65_536, hard: 65_536 },
+    ];
+    const deps = createDeps();
+    vi.mocked(deps.runOpenshell).mockImplementation((args: string[]) => {
+      if (args[0] === "sandbox" && args[1] === "get") {
+        return {
+          status: 0,
+          stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n",
+          stderr: "",
+        };
+      }
+      if (args[0] === "sandbox" && args[1] === "exec") {
+        return { status: 1, stdout: "", stderr: "permission denied" };
+      }
+      return { status: 0, stdout: "", stderr: "" };
+    });
+    mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce((options) => {
+      expect(options.checkReadyIdentity?.()).toBe("probe_failed");
+      return {
+        ready: false,
+        reason: "identity_probe_failed",
+        failurePhase: null,
+      };
+    });
+    mockExit();
+
+    await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1");
+
+    expect(patch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce();
+    expect(deps.runOpenshell).not.toHaveBeenCalledWith(
+      ["sandbox", "delete", "alpha"],
+      expect.anything(),
+    );
+    expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", {
+      backupPath: null,
+    });
+    expect(errorOutput()).toContain(
+      "NemoClaw left the sandbox in place for inspection and recovery",
+    );
+  });
+
+  it("keeps a transient recreated-sandbox not-ready response inside the readiness wait (#9050)", async () => {
+    const input = createInput();
+    const patch = createPatch();
+    mocks.createDockerGpuSandboxCreatePatch.mockReturnValueOnce(patch);
+    input.sandboxGpuConfig = {
+      ...input.sandboxGpuConfig,
+      mode: "0",
+      sandboxGpuEnabled: false,
+    };
+    input.gpuRoutePlan = "none";
+    input.initialGpuRoute = "none";
+    input.createArgv = ["openshell", "sandbox", "create"];
+    input.persistStartupCommand = true;
+    input.requiredUlimits = [
+      { name: "nproc", soft: 512, hard: 512 },
+      { name: "nofile", soft: 65_536, hard: 65_536 },
+    ];
+    const deps = createDeps();
+    let execAttempts = 0;
+    vi.mocked(deps.runOpenshell).mockImplementation((args: string[]) => {
+      if (args[0] === "sandbox" && args[1] === "get") {
+        return {
+          status: 0,
+          stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n",
+          stderr: "",
+        };
+      }
+      if (args[0] === "sandbox" && args[1] === "exec") {
+        execAttempts += 1;
+        return execAttempts === 1
+          ? {
+              status: 1,
+              stdout: "",
+              stderr:
+                `Error:   × code: 'The system is not in a state required for the operation's\n` +
+                '  │ execution\', message: "sandbox is not ready"\n',
+            }
+          : { status: 0, stdout: "", stderr: "" };
+      }
+      return { status: 0, stdout: "", stderr: "" };
+    });
+    mocks.waitForCreatedSandboxReadyWithTrace.mockImplementationOnce((options) => {
+      expect(options.checkReadyIdentity?.()).toBe("not_ready");
+      expect(options.checkReadyIdentity?.()).toBe("ready");
+      return { ready: true, reason: "ready", failurePhase: null };
+    });
+
+    await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({
+      route: "none",
+    });
+
+    expect(execAttempts).toBe(2);
+    expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled();
+    expect(deps.runOpenshell).not.toHaveBeenCalledWith(
+      ["sandbox", "delete", "alpha"],
+      expect.anything(),
+    );
   });
 
   it("does not replace a native GPU container solely to persist its startup command", async () => {

src/lib/onboard/sandbox-gpu-create-run-attempt.ts

@@ -1,7 +1,10 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { resolveOpenShellSandboxId } from "../adapters/openshell/sandbox-identity";
+import {
+  parseOpenShellSandboxId,
+  resolveOpenShellSandboxId,
+} from "../adapters/openshell/sandbox-identity";
 import { printSandboxCreateRecoveryHints } from "../build-context";
 import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy";
 import { streamSandboxCreate } from "../sandbox/create-stream";
@@ -27,6 +30,7 @@ import type {
   SandboxGpuCreateFlowInput,
 } from "./sandbox-gpu-create-flow";
 import * as sandboxGpuPreflight from "./sandbox-gpu-preflight";
+import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing";
 import * as sandboxReadinessTracing from "./sandbox-readiness-tracing";
 import { addTraceEvent } from "./tracing";
 
@@ -44,6 +48,61 @@ export type SandboxGpuCreateAttemptState = {
 // to live validation or the GPU proof.
 const REPLACEMENT_STABLE_READY_POLLS = 2;
 
+const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/gu;
+const OPENSHELL_SANDBOX_NOT_READY =
+  /^Error: code: 'The system is not in a state required for the operation's execution', message: "sandbox is not ready"$/iu;
+
+type OpenShellCommandResult = ReturnType<SandboxGpuCreateFlowDeps["runOpenshell"]>;
+
+function normalizedOpenShellCommandOutput(result: OpenShellCommandResult): string {
+  return `${String(result.stderr ?? "")}\n${String(result.stdout ?? "")}`
+    .replace(ANSI_RE, "")
+    .replace(/[×│]/gu, " ")
+    .replace(/\s+/gu, " ")
+    .trim();
+}
+
+type OpenShellSandboxIdentityProbe =
+  | { state: "identified"; sandboxId: string }
+  | { state: "not_ready" }
+  | { state: "failed" };
+
+function probeExactOpenShellSandboxId(
+  sandboxName: string,
+  deps: SandboxGpuCreateFlowDeps,
+): OpenShellSandboxIdentityProbe {
+  const result = deps.runOpenshell(["sandbox", "get", sandboxName], {
+    ignoreError: true,
+    suppressOutput: true,
+  });
+  if (result.status === 0 && !result.error) {
+    const sandboxId = parseOpenShellSandboxId(String(result.stdout ?? ""));
+    return sandboxId ? { state: "identified", sandboxId } : { state: "failed" };
+  }
+  return OPENSHELL_SANDBOX_NOT_READY.test(normalizedOpenShellCommandOutput(result))
+    ? { state: "not_ready" }
+    : { state: "failed" };
+}
+
+function checkRecreatedSandboxReadyIdentity(
+  sandboxName: string,
+  expectedSandboxId: string,
+  deps: SandboxGpuCreateFlowDeps,
+): ReturnType<CreatedSandboxReadyIdentityCheck> {
+  const identity = probeExactOpenShellSandboxId(sandboxName, deps);
+  if (identity.state === "not_ready") return "not_ready";
+  if (identity.state === "failed") return "probe_failed";
+  if (identity.sandboxId !== expectedSandboxId) return "identity_changed";
+  const result = deps.runOpenshell(["sandbox", "exec", "--name", sandboxName, "--", "true"], {
+    ignoreError: true,
+    suppressOutput: true,
+  });
+  if (result.status === 0 && !result.error) return "ready";
+  return OPENSHELL_SANDBOX_NOT_READY.test(normalizedOpenShellCommandOutput(result))
+    ? "not_ready"
+    : "probe_failed";
+}
+
 class ManagedBootstrapCreateStreamFailure extends Error {
   constructor(readonly result: Awaited<ReturnType<typeof streamSandboxCreate>>) {
     super("Managed bootstrap held workload did not complete its create stream.");
@@ -334,6 +393,21 @@ export function createSandboxGpuCreateAttemptRunner(
         );
       }
     }
+    const preRecreateIdentity = deferRestartSafeCutover
+      ? probeExactOpenShellSandboxId(input.sandboxName, deps)
+      : null;
+    const expectedRecreatedSandboxId =
+      preRecreateIdentity?.state === "identified" ? preRecreateIdentity.sandboxId : null;
+    if (deferRestartSafeCutover && !expectedRecreatedSandboxId) {
+      console.error("");
+      console.error(
+        `  Sandbox '${input.sandboxName}' reached Ready, but OpenShell did not return one exact durable sandbox ID before runtime recreation.`,
+      );
+      printSandboxCreateFailureDiagnostics(input.sandboxName, {
+        backupPath: input.restoreBackupPath,
+      });
+      process.exit(createResult.status === 0 ? 1 : createResult.status);
+    }
     await runtimePatch.ensureApplied();
     await runtimePatch.waitForSupervisorReconnectIfNeeded();
     console.log("  Waiting for sandbox to become ready...");
@@ -343,7 +417,14 @@ export function createSandboxGpuCreateAttemptRunner(
       runCaptureOpenshell: deps.runCaptureOpenshell,
       isSandboxReady,
       getSandboxFailurePhase,
-      stableReadyPolls: compatibility || managedBootstrap ? REPLACEMENT_STABLE_READY_POLLS : 1,
+      stableReadyPolls:
+        compatibility || managedBootstrap || expectedRecreatedSandboxId
+          ? REPLACEMENT_STABLE_READY_POLLS
+          : 1,
+      checkReadyIdentity: expectedRecreatedSandboxId
+        ? () =>
+            checkRecreatedSandboxReadyIdentity(input.sandboxName, expectedRecreatedSandboxId, deps)
+        : undefined,
       sleep: deps.sleep,
     });
     if (!readiness.ready) {
@@ -388,7 +469,11 @@ export function createSandboxGpuCreateAttemptRunner(
         backupPath: input.restoreBackupPath,
       });
       if (compatibility) runtimePatch.printReadinessFailureIfEnabled();
-      else {
+      else if (expectedRecreatedSandboxId) {
+        console.error(
+          "  NemoClaw did not start dashboard forwarding. NemoClaw left the sandbox in place for inspection and recovery.",
+        );
+      } else {
         const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], {
           ignoreError: true,
           suppressOutput: true,

src/lib/onboard/sandbox-readiness-tracing.test.ts

@@ -86,6 +86,64 @@ describe("createSandboxReadyWaiter", () => {
 });
 
 describe("waitForCreatedSandboxReadyWithTrace terminal-phase handling", () => {
+  it("waits for the exact recreated sandbox to become executable before accepting stable Ready (#9050)", () => {
+    const { runCaptureOpenshell, sleep } = replay([`${NAME}   Ready`]);
+    const checkReadyIdentity = vi.fn().mockReturnValueOnce("not_ready").mockReturnValue("ready");
+
+    expect(
+      waitForCreatedSandboxReadyWithTrace({
+        sandboxName: NAME,
+        timeoutSecs: 30,
+        runCaptureOpenshell,
+        isSandboxReady,
+        stableReadyPolls: 2,
+        checkReadyIdentity,
+        sleep,
+      }),
+    ).toEqual({ ready: true, reason: "ready", failurePhase: null });
+    expect(checkReadyIdentity).toHaveBeenCalledTimes(3);
+    expect(runCaptureOpenshell).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("stops when the recreated sandbox identity changes (#9050)", () => {
+    const { runCaptureOpenshell, sleep } = replay([`${NAME}   Ready`]);
+
+    expect(
+      waitForCreatedSandboxReadyWithTrace({
+        sandboxName: NAME,
+        timeoutSecs: 30,
+        runCaptureOpenshell,
+        isSandboxReady,
+        checkReadyIdentity: () => "identity_changed",
+        sleep,
+      }),
+    ).toEqual({ ready: false, reason: "identity_changed", failurePhase: null });
+    expect(runCaptureOpenshell).toHaveBeenCalledOnce();
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("stops after an unknown recreated-runtime probe failure (#9050)", () => {
+    const { runCaptureOpenshell, sleep } = replay([`${NAME}   Ready`]);
+
+    expect(
+      waitForCreatedSandboxReadyWithTrace({
+        sandboxName: NAME,
+        timeoutSecs: 30,
+        runCaptureOpenshell,
+        isSandboxReady,
+        checkReadyIdentity: () => "probe_failed",
+        sleep,
+      }),
+    ).toEqual({
+      ready: false,
+      reason: "identity_probe_failed",
+      failurePhase: null,
+    });
+    expect(runCaptureOpenshell).toHaveBeenCalledOnce();
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
   it("does not probe when the readiness deadline is zero (#3768)", () => {
     const { runCaptureOpenshell, sleep } = replay([`${NAME}   Ready`]);