fix(snapshot): explain unregistered clone recovery

docs/manage-sandboxes/backup-restore.mdx

@@ -196,6 +196,20 @@ For a stopped source, its registry entry must record both the sandbox image and
 NemoClaw stops before creating or replacing the destination when either record is missing, and directs you to run `$$nemoclaw onboard` when no image is recorded.
 For a Kubernetes-driver source, the pod image must remain resolvable through its gateway.
 
+For a new destination, NemoClaw waits for the owning gateway to report the sandbox as Ready with a valid live identity.
+It checks that identity again immediately before registration.
+NemoClaw assigns the destination a new lifecycle generation instead of copying the source sandbox's generation.
+
+If the destination is not Ready with the same valid identity, the restore exits nonzero before registration or state restore.
+The OpenShell sandbox remains created but unregistered, so `--force` cannot select it for deletion.
+Run the exact owner-scoped deletion command printed by the failure:
+
+```bash
+openshell sandbox delete -g '<owning-gateway>' '<destination>'
+```
+
+After OpenShell deletes the destination, rerun the original `snapshot restore --to` command.
+
 For dashboard-enabled agents, NemoClaw allocates the destination sandbox its own dashboard port instead of reusing the source port.
 If no port is available, restore stops before deleting an existing `--force` destination.
 

docs/reference/commands.mdx

@@ -3606,6 +3606,18 @@ When `dst` does not exist, it is auto-created from the source image.
 For a Docker- or VM-driver source, the source can be stopped when its registry entry records both the sandbox image and a complete inference route.
 For a Kubernetes-driver source, the pod image must remain resolvable through its gateway.
 No re-onboarding is needed when those prerequisites are present.
+For a new destination, NemoClaw requires its owning gateway to report Ready state and a valid live identity.
+It revalidates that identity immediately before registration.
+The destination receives a new lifecycle generation and does not inherit the source sandbox's generation.
+If the destination is not Ready with the same valid identity, the command exits nonzero before registration or state restore.
+The created destination remains unregistered, so `--force` cannot select it for deletion.
+Run the exact owner-scoped deletion command printed by the failure:
+
+```bash
+openshell sandbox delete -g '<owning-gateway>' '<destination>'
+```
+
+After OpenShell deletes the destination, rerun the original `snapshot restore --to` command.
 <AgentOnly variant="openclaw">
 After NemoClaw creates the destination, it waits for the managed OpenClaw supervisor to pass a bounded readiness check before it applies snapshot state.
 If the check fails, the command leaves the destination registered without restored snapshot state and exits nonzero.

src/lib/actions/sandbox/snapshot.ts

@@ -47,7 +47,7 @@ import {
 } from "../../onboard/observability-policy-presets";
 import { normalizePolicyTierName } from "../../onboard/policy-tier-suppression";
 import * as policies from "../../policy";
-import { ROOT, run, validateName } from "../../runner";
+import { ROOT, run, shellQuote, validateName } from "../../runner";
 import { parseLiveSandboxNames } from "../../runtime-recovery";
 import { streamSandboxCreate } from "../../sandbox/create-stream";
 import * as shields from "../../shields";
@@ -140,6 +140,16 @@ function snapshotExit(exitCode = 1): never {
   throw new SnapshotCommandError([], exitCode);
 }
 
+function failUnregisteredSnapshotClone(sandboxName: string, gatewayName: string): never {
+  throw new SnapshotCommandError([
+    `  Sandbox '${sandboxName}' was created, but its owning gateway did not provide a valid Ready identity.`,
+    "  Snapshot state was not restored and the clone was not registered.",
+    "  Remove the unregistered sandbox before retrying:",
+    `    openshell sandbox delete -g ${shellQuote(gatewayName)} ${shellQuote(sandboxName)}`,
+    "  Then rerun the original snapshot restore command.",
+  ]);
+}
+
 function formatSnapshotVersion(b: unknown) {
   const snapshotVersion = (b as { snapshotVersion?: number }).snapshotVersion ?? 0;
   return `v${snapshotVersion}`;
@@ -475,10 +485,14 @@ async function autoCreateSandboxFromSource(
     ignoreError: true,
   });
   if (verify.status !== 0 || !isSandboxReady(verify.output || "", dstName)) {
-    console.error(`  Sandbox '${dstName}' did not reach Ready state after create.`);
-    snapshotExit(1);
+    failUnregisteredSnapshotClone(dstName, sourceGatewayName);
+  }
+  let lifecycleRegistration: ReturnType<typeof cloneLifecycle.capture>;
+  try {
+    lifecycleRegistration = cloneLifecycle.capture();
+  } catch {
+    failUnregisteredSnapshotClone(dstName, sourceGatewayName);
   }
-  const lifecycleRegistration = cloneLifecycle.capture();
 
   // DNS proxy is only meaningful for the kubernetes driver (matches onboard.ts).
   const dnsScript = path.join(ROOT, "scripts", "setup-dns-proxy.sh");
@@ -493,6 +507,12 @@ async function autoCreateSandboxFromSource(
   // Register dst in the NemoClaw registry, cloning most fields from src.
   // Policies are cleared here — the caller replays them from the snapshot
   // manifest after the restore succeeds and writes them back into this entry.
+  let finalLifecycleRegistration: ReturnType<typeof cloneLifecycle.revalidate>;
+  try {
+    finalLifecycleRegistration = cloneLifecycle.revalidate(lifecycleRegistration);
+  } catch {
+    failUnregisteredSnapshotClone(dstName, sourceGatewayName);
+  }
   registry.registerSandbox({
     ...srcEntry,
     name: dstName,
@@ -521,7 +541,7 @@ async function autoCreateSandboxFromSource(
     // stop/start, recovery, and later snapshots can address its gateway.
     gatewayName: sourceGatewayName,
     gatewayPort: sourceGatewayPort,
-    ...cloneLifecycle.revalidate(lifecycleRegistration),
+    ...finalLifecycleRegistration,
   });
 
   const sourceAgent = (srcEntry as SandboxEntry).agent || "openclaw";

src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts

@@ -62,7 +62,9 @@ it("journals not-ready repair on the selected non-default gateway (#6492)", asyn
       reference: "openshell/sandbox-from:new",
     },
     lifecycleGeneration: "replacement-generation",
-    lifecycleLiveIdentityFingerprint: "replacement-identity",
+    lifecycleLiveIdentityFingerprint: fingerprintSandboxRecreateValue(
+      "openshell-replacement-id",
+    ),
   };
   const phases: Array<string | null> = [];
   const updateSession = vi.fn((mutator: (value: Session) => Session | void) => {

src/lib/onboard/sandbox-recreate-transaction.test.ts

@@ -404,7 +404,7 @@ describe("sandbox recreate journal", () => {
     });
   });
 
-  it("rejects a malformed identity at the lower runtime boundary (#8942)", () => {
+  it("rejects a malformed replacement identity before the recreate journal records it (#8942)", () => {
     const session = createSession({ sandboxName: "alpha" });
     beginSandboxRecreateTransaction(
       session,
@@ -444,7 +444,7 @@ describe("sandbox recreate journal", () => {
         state: "ready",
         liveIdentityFingerprint: "not-a-fingerprint",
       }),
-    ).toThrow(/stable OpenShell Id/u);
+    ).toThrow(/valid live identity fingerprint/u);
     expect(session.checkpoint?.sandboxRecreate).toMatchObject({
       phase: "creating",
       targetLiveIdentityFingerprint: null,

src/lib/onboard/sandbox-recreate-transaction.ts

@@ -593,7 +593,9 @@ export function recordSandboxRecreateTargetCreated(
     !observation.liveIdentityFingerprint ||
     !/^[0-9a-f]{64}$/u.test(observation.liveIdentityFingerprint)
   ) {
-    throw new Error("The journaled replacement must be ready with a stable OpenShell Id.");
+    throw new Error(
+      "The journaled replacement must be Ready with a valid live identity fingerprint.",
+    );
   }
   const checkpoint = baseCheckpoint(session);
   const current = checkpoint.sandboxRecreate;

test/snapshot-gateway-guard.test.ts

@@ -215,6 +215,8 @@ function makeVmRestoreToEnv(
   prefix: string,
   entry: Record<string, unknown> = { imageTag: "openshell/sandbox-from:fast-path-test" },
   cloneIdentity = "fixture-clone-1",
+  cloneReady = true,
+  revalidatedCloneIdentity = cloneIdentity,
 ): Record<string, string> {
   const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
   const localBin = path.join(home, "bin");
@@ -230,20 +232,22 @@ function makeVmRestoreToEnv(
 
   const cloneReadyMarker = path.join(home, "clone-1-ready");
   const cloneRunningMarker = path.join(home, "clone-1-running");
+  const cloneIdentityCapturedMarker = path.join(home, "clone-1-identity-captured");
+  const markCloneReady = cloneReady ? `touch ${JSON.stringify(cloneReadyMarker)}` : ":";
   const gatewayLifecycleLog = path.join(home, "gateway-lifecycle.log");
   const dashboardBind = process.env.WSL_DISTRO_NAME ? "0.0.0.0" : "127.0.0.1";
   writeExecutable(path.join(localBin, "openshell"), [
     'case "$1 $2" in',
     '  "gateway info") printf "Gateway Info\\n\\nGateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080/\\n"; exit 0 ;;',
-    `  "sandbox get") [ "$3 $4" = "-g nemoclaw" ] || exit 91; for sandbox_ref in "$@"; do :; done; printf "Name: %s\\nId: %s\\nPhase: Ready\\n" "$sandbox_ref" ${JSON.stringify(cloneIdentity)}; exit 0 ;;`,
+    `  "sandbox get") [ "$3 $4" = "-g nemoclaw" ] || exit 91; for sandbox_ref in "$@"; do :; done; if [ -f ${JSON.stringify(cloneIdentityCapturedMarker)} ]; then clone_identity=${JSON.stringify(revalidatedCloneIdentity)}; else touch ${JSON.stringify(cloneIdentityCapturedMarker)}; clone_identity=${JSON.stringify(cloneIdentity)}; fi; printf "Name: %s\\nId: %s\\nPhase: Ready\\n" "$sandbox_ref" "$clone_identity"; exit 0 ;;`,
     `  "sandbox list") if [ -n "\${3:-}" ] && [ "$3 $4" != "-g nemoclaw" ]; then exit 91; fi; if [ -f ${JSON.stringify(cloneReadyMarker)} ]; then printf "NAME STATUS\\nalpha Ready\\nclone-1 Ready\\n"; else printf "NAME STATUS\\nalpha Ready\\n"; fi; exit 0 ;;`,
     '  "sandbox exec")',
     '    case "$*" in',
     '      *"__NEMOCLAW_SANDBOX_EXEC_STARTED__"*) printf "__NEMOCLAW_SANDBOX_EXEC_STARTED__\\nRUNNING\\n"; exit 0 ;;',
     "    esac",
     '    printf "NEMOCLAW_DCODE_PROBE=no-runtime\\n"; exit 0 ;;',
     '  "sandbox ssh-config") for sandbox_ref in "$@"; do :; done; printf "Host openshell-%s\\n  HostName 127.0.0.1\\n  User sandbox\\n" "$sandbox_ref"; exit 0 ;;',
-    `  "sandbox create") touch ${JSON.stringify(cloneReadyMarker)} ${JSON.stringify(cloneRunningMarker)}; printf "created clone-1\\n"; exit 0 ;;`,
+    `  "sandbox create") ${markCloneReady}; touch ${JSON.stringify(cloneRunningMarker)}; printf "created clone-1\\n"; exit 0 ;;`,
     `  "forward list") printf "SANDBOX BIND PORT PID STATUS\\nclone-1 ${dashboardBind} ${String(dashboardPort)} 4242 running\\n"; exit 0 ;;`,
     '  "forward stop") exit 1 ;;',
     "esac",
@@ -252,13 +256,15 @@ function makeVmRestoreToEnv(
   ]);
 
   const remoteOpenClawJson = path.join(home, "remote-openclaw.json");
+  const snapshotRestoreMarker = path.join(home, "snapshot-restore-attempted");
   fs.writeFileSync(remoteOpenClawJson, JSON.stringify({ gateway: { auth: { token: "fresh" } } }));
   writeExecutable(path.join(localBin, "ssh"), [
     `REMOTE_OPENCLAW_JSON=${JSON.stringify(remoteOpenClawJson)}`,
+    `SNAPSHOT_RESTORE_MARKER=${JSON.stringify(snapshotRestoreMarker)}`,
     'cmd=""; for arg do cmd="$arg"; done',
     'if printf "%s" "$cmd" | grep -q "openclaw.json"; then',
     '  if printf "%s" "$cmd" | grep -q "cat --"; then cat "$REMOTE_OPENCLAW_JSON"; exit 0; fi',
-    '  if printf "%s" "$cmd" | grep -q ".nemoclaw-restore"; then cat > "$REMOTE_OPENCLAW_JSON"; exit 0; fi',
+    '  if printf "%s" "$cmd" | grep -q ".nemoclaw-restore"; then touch "$SNAPSHOT_RESTORE_MARKER"; cat > "$REMOTE_OPENCLAW_JSON"; exit 0; fi',
     "fi",
     "exit 0",
   ]);
@@ -307,6 +313,7 @@ function makeVmRestoreToEnv(
   return {
     HOME: home,
     NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0",
+    NEMOCLAW_TEST_SNAPSHOT_RESTORE_MARKER: snapshotRestoreMarker,
     PATH: `${localBin}:${process.env.PATH ?? ""}`,
   };
 }
@@ -338,7 +345,7 @@ describe("snapshot VM-driver gateway guard", () => {
 
   // `snapshot restore --to <new>` on VM driver must use the registered
   // imageTag, not the legacy `docker exec ... kubectl` probe.
-  it("snapshot restore --to records a fresh clone lifecycle identity before pairing verification (#8942)", () => {
+  it("snapshot restore --to records a new clone lifecycle generation and live identity (#8942)", () => {
     const env = makeVmRestoreToEnv("nemoclaw-snap-vm-gw-restore-to-", {
       imageTag: "openshell/sandbox-from:fast-path-test",
       lifecycleGeneration: "source-generation",
@@ -382,13 +389,55 @@ describe("snapshot VM-driver gateway guard", () => {
 
     const r = runCli("alpha snapshot restore baseline --to clone-1", env);
     expect(r.code, r.out).toBe(1);
-    expect(r.out).toContain("valid live identity");
+    expect(r.out).toContain("owning gateway did not provide a valid Ready identity");
+    expect(r.out).toContain("Snapshot state was not restored and the clone was not registered.");
+    expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'");
     const registryState = JSON.parse(
       fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"),
     );
     expect(registryState.sandboxes["clone-1"]).toBeUndefined();
   }, 15000);
 
+  it("snapshot restore --to reports owner-scoped recovery when a created clone is not Ready (#8942)", () => {
+    const env = makeVmRestoreToEnv(
+      "nemoclaw-snap-vm-gw-restore-to-not-ready-",
+      { imageTag: "openshell/sandbox-from:fast-path-test" },
+      "fixture-clone-1",
+      false,
+    );
+
+    const r = runCli("alpha snapshot restore baseline --to clone-1", env);
+    expect(r.code, r.out).toBe(1);
+    expect(r.out).toContain("owning gateway did not provide a valid Ready identity");
+    expect(r.out).toContain("Snapshot state was not restored and the clone was not registered.");
+    expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'");
+    const registryState = JSON.parse(
+      fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"),
+    );
+    expect(registryState.sandboxes["clone-1"]).toBeUndefined();
+  }, 15000);
+
+  it("snapshot restore --to reports owner-scoped recovery when clone identity changes before registration (#8942)", () => {
+    const env = makeVmRestoreToEnv(
+      "nemoclaw-snap-vm-gw-restore-to-identity-drift-",
+      { imageTag: "openshell/sandbox-from:fast-path-test" },
+      "fixture-clone-1",
+      true,
+      "fixture-clone-2",
+    );
+
+    const r = runCli("alpha snapshot restore baseline --to clone-1", env);
+    expect(r.code, r.out).toBe(1);
+    expect(r.out).toContain("owning gateway did not provide a valid Ready identity");
+    expect(r.out).toContain("Snapshot state was not restored and the clone was not registered.");
+    expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'");
+    const registryState = JSON.parse(
+      fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"),
+    );
+    expect(registryState.sandboxes["clone-1"]).toBeUndefined();
+    expect(fs.existsSync(env.NEMOCLAW_TEST_SNAPSHOT_RESTORE_MARKER)).toBe(false);
+  }, 15000);
+
   it("snapshot restore --to fails closed for VM-driver entries missing imageTag", () => {
     const env = makeVmRestoreToEnv("nemoclaw-snap-vm-gw-restore-to-missing-image-", {
       imageTag: null,