fix(recovery): close post-merge failure gaps

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -27,7 +27,7 @@ If Docker no longer has the container, follow the printed `rebuild --yes` guidan
 
 <AgentOnly variant="openclaw,hermes">
 The `start` command returns success only after it authenticates the recovered agent runtime, OpenShell reports the sandbox ready, and host-side port forwards pass their checks.
-If any check fails, the command keeps the existing container, exits nonzero, identifies the failure, and tells you to run `recover` before retrying `start`.
+If a check fails, the command exits nonzero, identifies the failure, and prints recovery guidance before you retry `start`.
 </AgentOnly>
 
 <AgentOnly variant="openclaw">
@@ -135,6 +135,7 @@ After a transactional recreation, NemoClaw waits 120 seconds for OpenShell to re
 Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` before the recovery command to change this budget.
 A definitive managed-health failure still stops immediately.
 If re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement and leaves the primary dashboard or API host forward stopped.
+If NemoClaw cannot confirm rollback to the previous container, inspect Docker state before you retry recovery.
 
 For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control).
 If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it.

src/lib/actions/sandbox/connect-flow.test.ts

@@ -972,6 +972,77 @@ describe("connectSandbox flow", () => {
     expect(exitSpy).toHaveBeenCalledWith(1);
   });
 
+  it.each([
+    {
+      condition: "a Docker final-handoff failure",
+      expectedDetail:
+        "Docker could not start the replacement container to complete the final recovery handoff",
+      recoveryFailureDetail:
+        "Docker could not start the replacement container to complete the final recovery handoff",
+    },
+    {
+      condition: "a pinned replacement identity failure",
+      expectedDetail:
+        "the replacement container identity changed during the final managed supervisor health check",
+      recoveryFailureDetail:
+        "the replacement container identity changed during the final managed supervisor health check",
+    },
+    {
+      condition: "an OpenShell readiness failure",
+      expectedDetail: "the replacement container did not become ready in OpenShell",
+      recoveryFailureDetail:
+        "the replacement container did not become ready in OpenShell\nAuthorization: Bearer opaque-connect-recovery-token\u001b[31m",
+    },
+    {
+      condition: "an unconfirmed rollback after a gateway wait failure",
+      expectedDetail: "NemoClaw could not confirm rollback to the previous sandbox container",
+      recoveryFailureDetail:
+        "NemoClaw could not confirm rollback to the previous sandbox container. Inspect Docker state before retrying. Recovery failure before rollback: the recovered gateway did not become responsive before the recovery timeout",
+    },
+    {
+      condition: "a detail-free recovery failure",
+      expectedDetail: "the gateway recovery attempt did not complete",
+      recoveryFailureDetail: undefined,
+    },
+  ])(
+    "stops non-probe connect before route repair, pairing, or SSH after $condition (#9364)",
+    async ({ expectedDetail, recoveryFailureDetail }) => {
+      const harness = createConnectHarness({
+        registryEntry: { model: "qwen3-vl:4b", provider: "ollama-local" },
+        processCheck: {
+          checked: true,
+          wasRunning: false,
+          recovered: false,
+          forwardRecovered: false,
+          recoveryFailureDetail,
+        },
+      });
+
+      await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)");
+
+      const errorOutput = harness.errorSpy.mock.calls
+        .map((call) => String(call[0] ?? ""))
+        .join("\n");
+      expect(errorOutput).toContain(
+        "Recovery failed: NemoClaw could not recover the OpenClaw gateway in 'alpha'",
+      );
+      expect(errorOutput).toContain(expectedDetail);
+      expect(errorOutput).not.toContain("opaque-connect-recovery-token");
+      expect(errorOutput).not.toContain("\u001b");
+      expect(harness.ensureOllamaAuthProxySpy).not.toHaveBeenCalled();
+      expect(harness.findReachableOllamaHostSpy).not.toHaveBeenCalled();
+      expect(harness.withGatewayRouteMutationLockSpy).not.toHaveBeenCalled();
+      expect(harness.settlePortablePairingSpy).not.toHaveBeenCalled();
+      expect(harness.runAutoPairSpy).not.toHaveBeenCalled();
+      expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith(
+        "openshell",
+        ["sandbox", "connect", "alpha"],
+        expect.any(Object),
+      );
+      expect(exitSpy).toHaveBeenCalledWith(1);
+    },
+  );
+
   it("redacts untrusted gateway recovery details before reporting them", async () => {
     const opaqueToken = "opaque-gateway-recovery-token";
     const harness = createConnectHarness({

src/lib/actions/sandbox/connect.ts

@@ -267,14 +267,20 @@ function exitOnGatewayRecoveryFailure(
   sandboxName: string,
   agentName: string,
   detail: string,
+  operation: "Probe" | "Recovery" = "Probe",
+  showWedgeDiagnostics = false,
 ): never {
   const safeDetail = sanitizeSandboxStartupRecoveryDetail(detail);
   const terminalPunctuation = /[.!?]$/u.test(safeDetail) ? "" : ".";
   console.error("");
   console.error(
-    `  Probe failed: NemoClaw could not recover the ${agentName} gateway in '${sandboxName}'.`,
+    `  ${operation} failed: NemoClaw could not recover the ${agentName} gateway in '${sandboxName}'.`,
   );
   console.error(`  Recovery detail: ${safeDetail}${terminalPunctuation}`);
+  if (showWedgeDiagnostics) {
+    printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand);
+    console.error("  Check /tmp/gateway.log inside the sandbox for details.");
+  }
   process.exit(1);
 }
 
@@ -341,6 +347,8 @@ async function runSandboxConnectProbe(sandboxName: string): Promise<void> {
       sandboxName,
       agentName,
       String(processCheck.recoveryFailureDetail),
+      "Probe",
+      true,
     );
   }
   if (processCheck.wasRunning) {
@@ -1295,6 +1303,16 @@ export async function prepareInteractiveSession(
     const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName));
     exitOnMcpReconciliationRefusal(sandboxName, agentName, processCheck, "Connect");
   }
+  const recoveryFailureDetail =
+    "recoveryFailureDetail" in processCheck && processCheck.recoveryFailureDetail
+      ? String(processCheck.recoveryFailureDetail)
+      : processCheck.checked && processCheck.wasRunning === false && processCheck.recovered === false
+        ? "the gateway recovery attempt did not complete"
+        : null;
+  if (recoveryFailureDetail) {
+    const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName));
+    exitOnGatewayRecoveryFailure(sandboxName, agentName, recoveryFailureDetail, "Recovery");
+  }
   // Ensure Ollama auth proxy is running (recovers from host reboots)
   ensureOllamaAuthProxy();
 

src/lib/actions/sandbox/gateway-restart.test.ts

@@ -16,6 +16,7 @@ const supervisorFailureMarkers: Array<
   [string, ReturnType<typeof classifyGatewayRestartFailure>["layer"]]
 > = [
   ["PRIVILEGED_CONTROL_UNAVAILABLE", "privileged control unavailable"],
+  ["MANAGED_CONTROL_IDENTITY_CHANGED", "container identity changed"],
   ["SUPERVISOR_UNAVAILABLE", "privileged control unavailable"],
   ["SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=await-replacement", "supervisor unavailable"],
   ["SUPERVISOR_NOT_RUNNING", "supervisor not running"],
@@ -90,6 +91,25 @@ describe("gateway restart failure classification precedence", () => {
       layer: "supervisor not running",
     });
   });
+
+  it("does not classify an embedded identity marker as a protocol marker", () => {
+    expect(classify("failure mentions MANAGED_CONTROL_IDENTITY_CHANGED inline")).toMatchObject({
+      layer: "launch failure",
+    });
+  });
+
+  it("removes every complete identity marker line from the failure detail", () => {
+    const output = [
+      " MANAGED_CONTROL_IDENTITY_CHANGED ",
+      "container changed once",
+      "MANAGED_CONTROL_IDENTITY_CHANGED",
+      "container changed again",
+    ].join("\n");
+    expect(classify(output)).toEqual({
+      layer: "container identity changed",
+      detail: "container changed once\ncontainer changed again",
+    });
+  });
 });
 
 describe("restartSandboxGateway — host-mediated gateway restart", () => {

src/lib/actions/sandbox/gateway-restart.ts

@@ -15,6 +15,8 @@ export type GatewayRestartCommandResult = {
   stderr: string;
 };
 
+export const MANAGED_CONTROL_IDENTITY_CHANGED_MARKER = "MANAGED_CONTROL_IDENTITY_CHANGED";
+
 export type ManagedGatewayControlCompletion = {
   disposition: "ok" | "already-running";
   oldPid: number;
@@ -49,6 +51,7 @@ export type GatewayRestartFailureLayer =
   | "privileged control unavailable"
   | "supervisor not running"
   | "supervisor unavailable"
+  | "container identity changed"
   | "secret-boundary refusal"
   | "unsafe config path"
   | "config hash mismatch"
@@ -185,6 +188,10 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul
   }
 
   const output = gatewayRestartOutput(result);
+  const outputLines = output.split(/\r?\n/);
+  const isIdentityChangedMarkerLine = (line: string) =>
+    line.trim() === MANAGED_CONTROL_IDENTITY_CHANGED_MARKER;
+  const hasIdentityChangedMarker = outputLines.some(isIdentityChangedMarkerLine);
   const detail = sanitizeGatewayRestartFailureDetail(output.trim());
   if (output.includes("SUPERVISOR_NOT_RUNNING")) {
     return {
@@ -198,6 +205,15 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul
       detail: detail || "the managed gateway supervisor became unavailable",
     };
   }
+  if (hasIdentityChangedMarker) {
+    return {
+      layer: "container identity changed",
+      detail:
+        sanitizeGatewayRestartFailureDetail(
+          outputLines.filter((line) => !isIdentityChangedMarkerLine(line)).join("\n").trim(),
+        ) || "the selected container identity changed",
+    };
+  }
   if (
     output.includes(MARKERS.ROOT_EXEC_UNAVAILABLE) ||
     output.includes("PRIVILEGED_CONTROL_UNAVAILABLE") ||

src/lib/actions/sandbox/launch.test.ts

@@ -461,6 +461,27 @@ describe("launchSandbox", () => {
     expect(mocks.publishLaunchReadiness).toHaveBeenCalledBefore(mocks.execSandbox);
   });
 
+  it("stops before readiness publication and agent execution when recovery rejects launch (#9364)", async () => {
+    mocks.inspectLaunchReadiness.mockResolvedValue({
+      kind: "fallback",
+      category: "expired",
+      fence: { epochId: "a".repeat(64) },
+      gatewayName: "nemoclaw",
+      gatewayPort: 8080,
+      fenceFailed: false,
+      recoveryBlocked: false,
+    });
+    const recoveryFailure = new Error("process.exit(1)");
+    mocks.prepareInteractiveSession.mockRejectedValueOnce(recoveryFailure);
+
+    await expect(launchSandbox("alpha")).rejects.toBe(recoveryFailure);
+
+    expect(mocks.prepareInteractiveSession).toHaveBeenCalledOnce();
+    expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled();
+    expect(mocks.prepareHermesLightTerminalSkin).not.toHaveBeenCalled();
+    expect(mocks.execSandbox).not.toHaveBeenCalled();
+  });
+
   it("keeps ordinary launch available when evidence observation, hashing, or storage fails (#8942)", async () => {
     mocks.inspectLaunchReadiness.mockResolvedValue({
       kind: "fallback",