diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 2e0b6f6d501..72cd7fa3441 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -196,6 +196,20 @@ For a stopped source, its registry entry must record both the sandbox image and NemoClaw stops before creating or replacing the destination when either record is missing, and directs you to run `$$nemoclaw onboard` when no image is recorded. For a Kubernetes-driver source, the pod image must remain resolvable through its gateway. +For a new destination, NemoClaw waits for the owning gateway to report the sandbox as Ready with a valid live identity. +It checks that identity again immediately before registration. +NemoClaw assigns the destination a new lifecycle generation instead of copying the source sandbox's generation. + +If the destination is not Ready with the same valid identity, the restore exits nonzero before registration or state restore. +The OpenShell sandbox remains created but unregistered, so `--force` cannot select it for deletion. +Run the exact owner-scoped deletion command printed by the failure: + +```bash +openshell sandbox delete -g '' '' +``` + +After OpenShell deletes the destination, rerun the original `snapshot restore --to` command. + For dashboard-enabled agents, NemoClaw allocates the destination sandbox its own dashboard port instead of reusing the source port. If no port is available, restore stops before deleting an existing `--force` destination. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f15b934af53..b4152665886 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3606,6 +3606,18 @@ When `dst` does not exist, it is auto-created from the source image. For a Docker- or VM-driver source, the source can be stopped when its registry entry records both the sandbox image and a complete inference route. For a Kubernetes-driver source, the pod image must remain resolvable through its gateway. No re-onboarding is needed when those prerequisites are present. +For a new destination, NemoClaw requires its owning gateway to report Ready state and a valid live identity. +It revalidates that identity immediately before registration. +The destination receives a new lifecycle generation and does not inherit the source sandbox's generation. +If the destination is not Ready with the same valid identity, the command exits nonzero before registration or state restore. +The created destination remains unregistered, so `--force` cannot select it for deletion. +Run the exact owner-scoped deletion command printed by the failure: + +```bash +openshell sandbox delete -g '' '' +``` + +After OpenShell deletes the destination, rerun the original `snapshot restore --to` command. After NemoClaw creates the destination, it waits for the managed OpenClaw supervisor to pass a bounded readiness check before it applies snapshot state. If the check fails, the command leaves the destination registered without restored snapshot state and exits nonzero. diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 4024b81d077..4d0e8902171 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -47,7 +47,7 @@ import { } from "../../onboard/observability-policy-presets"; import { normalizePolicyTierName } from "../../onboard/policy-tier-suppression"; import * as policies from "../../policy"; -import { ROOT, run, validateName } from "../../runner"; +import { ROOT, run, shellQuote, validateName } from "../../runner"; import { parseLiveSandboxNames } from "../../runtime-recovery"; import { streamSandboxCreate } from "../../sandbox/create-stream"; import * as shields from "../../shields"; @@ -140,6 +140,16 @@ function snapshotExit(exitCode = 1): never { throw new SnapshotCommandError([], exitCode); } +function failUnregisteredSnapshotClone(sandboxName: string, gatewayName: string): never { + throw new SnapshotCommandError([ + ` Sandbox '${sandboxName}' was created, but NemoClaw could not verify the same valid Ready identity from its owning gateway before registration.`, + " Snapshot state was not restored and the clone was not registered.", + " Remove the unregistered sandbox before retrying:", + ` openshell sandbox delete -g ${shellQuote(gatewayName)} ${shellQuote(sandboxName)}`, + " Then rerun the original snapshot restore command.", + ]); +} + function formatSnapshotVersion(b: unknown) { const snapshotVersion = (b as { snapshotVersion?: number }).snapshotVersion ?? 0; return `v${snapshotVersion}`; @@ -475,10 +485,14 @@ async function autoCreateSandboxFromSource( ignoreError: true, }); if (verify.status !== 0 || !isSandboxReady(verify.output || "", dstName)) { - console.error(` Sandbox '${dstName}' did not reach Ready state after create.`); - snapshotExit(1); + failUnregisteredSnapshotClone(dstName, sourceGatewayName); + } + let lifecycleRegistration: ReturnType; + try { + lifecycleRegistration = cloneLifecycle.capture(); + } catch { + failUnregisteredSnapshotClone(dstName, sourceGatewayName); } - const lifecycleRegistration = cloneLifecycle.capture(); // DNS proxy is only meaningful for the kubernetes driver (matches onboard.ts). const dnsScript = path.join(ROOT, "scripts", "setup-dns-proxy.sh"); @@ -493,6 +507,12 @@ async function autoCreateSandboxFromSource( // Register dst in the NemoClaw registry, cloning most fields from src. // Policies are cleared here — the caller replays them from the snapshot // manifest after the restore succeeds and writes them back into this entry. + let finalLifecycleRegistration: ReturnType; + try { + finalLifecycleRegistration = cloneLifecycle.revalidate(lifecycleRegistration); + } catch { + failUnregisteredSnapshotClone(dstName, sourceGatewayName); + } registry.registerSandbox({ ...srcEntry, name: dstName, @@ -521,7 +541,7 @@ async function autoCreateSandboxFromSource( // stop/start, recovery, and later snapshots can address its gateway. gatewayName: sourceGatewayName, gatewayPort: sourceGatewayPort, - ...cloneLifecycle.revalidate(lifecycleRegistration), + ...finalLifecycleRegistration, }); const sourceAgent = (srcEntry as SandboxEntry).agent || "openclaw"; diff --git a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts index e74784361b2..aefc5b1bb73 100644 --- a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts @@ -62,7 +62,9 @@ it("journals not-ready repair on the selected non-default gateway (#6492)", asyn reference: "openshell/sandbox-from:new", }, lifecycleGeneration: "replacement-generation", - lifecycleLiveIdentityFingerprint: "replacement-identity", + lifecycleLiveIdentityFingerprint: fingerprintSandboxRecreateValue( + "openshell-replacement-id", + ), }; const phases: Array = []; const updateSession = vi.fn((mutator: (value: Session) => Session | void) => { diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index c7bc3ac63bb..25928c0ec53 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -404,7 +404,7 @@ describe("sandbox recreate journal", () => { }); }); - it("rejects a malformed identity at the lower runtime boundary (#8942)", () => { + it("rejects a malformed replacement identity before the recreate journal records it (#8942)", () => { const session = createSession({ sandboxName: "alpha" }); beginSandboxRecreateTransaction( session, @@ -444,7 +444,7 @@ describe("sandbox recreate journal", () => { state: "ready", liveIdentityFingerprint: "not-a-fingerprint", }), - ).toThrow(/stable OpenShell Id/u); + ).toThrow(/valid live identity fingerprint/u); expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "creating", targetLiveIdentityFingerprint: null, diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 4320d646e60..01e246ebd7c 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -593,7 +593,9 @@ export function recordSandboxRecreateTargetCreated( !observation.liveIdentityFingerprint || !/^[0-9a-f]{64}$/u.test(observation.liveIdentityFingerprint) ) { - throw new Error("The journaled replacement must be ready with a stable OpenShell Id."); + throw new Error( + "The journaled replacement must be Ready with a valid live identity fingerprint.", + ); } const checkpoint = baseCheckpoint(session); const current = checkpoint.sandboxRecreate; diff --git a/test/snapshot-gateway-guard.test.ts b/test/snapshot-gateway-guard.test.ts index 836940aa033..f0d52565e14 100644 --- a/test/snapshot-gateway-guard.test.ts +++ b/test/snapshot-gateway-guard.test.ts @@ -215,6 +215,8 @@ function makeVmRestoreToEnv( prefix: string, entry: Record = { imageTag: "openshell/sandbox-from:fast-path-test" }, cloneIdentity = "fixture-clone-1", + cloneReady = true, + revalidatedCloneIdentity = cloneIdentity, ): Record { const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); const localBin = path.join(home, "bin"); @@ -230,12 +232,14 @@ function makeVmRestoreToEnv( const cloneReadyMarker = path.join(home, "clone-1-ready"); const cloneRunningMarker = path.join(home, "clone-1-running"); + const cloneIdentityCapturedMarker = path.join(home, "clone-1-identity-captured"); + const markCloneReady = cloneReady ? `touch ${JSON.stringify(cloneReadyMarker)}` : ":"; const gatewayLifecycleLog = path.join(home, "gateway-lifecycle.log"); const dashboardBind = process.env.WSL_DISTRO_NAME ? "0.0.0.0" : "127.0.0.1"; writeExecutable(path.join(localBin, "openshell"), [ 'case "$1 $2" in', ' "gateway info") printf "Gateway Info\\n\\nGateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080/\\n"; exit 0 ;;', - ` "sandbox get") [ "$3 $4" = "-g nemoclaw" ] || exit 91; for sandbox_ref in "$@"; do :; done; printf "Name: %s\\nId: %s\\nPhase: Ready\\n" "$sandbox_ref" ${JSON.stringify(cloneIdentity)}; exit 0 ;;`, + ` "sandbox get") [ "$3 $4" = "-g nemoclaw" ] || exit 91; for sandbox_ref in "$@"; do :; done; if [ -f ${JSON.stringify(cloneIdentityCapturedMarker)} ]; then clone_identity=${JSON.stringify(revalidatedCloneIdentity)}; else touch ${JSON.stringify(cloneIdentityCapturedMarker)}; clone_identity=${JSON.stringify(cloneIdentity)}; fi; printf "Name: %s\\nId: %s\\nPhase: Ready\\n" "$sandbox_ref" "$clone_identity"; exit 0 ;;`, ` "sandbox list") if [ -n "\${3:-}" ] && [ "$3 $4" != "-g nemoclaw" ]; then exit 91; fi; if [ -f ${JSON.stringify(cloneReadyMarker)} ]; then printf "NAME STATUS\\nalpha Ready\\nclone-1 Ready\\n"; else printf "NAME STATUS\\nalpha Ready\\n"; fi; exit 0 ;;`, ' "sandbox exec")', ' case "$*" in', @@ -243,7 +247,7 @@ function makeVmRestoreToEnv( " esac", ' printf "NEMOCLAW_DCODE_PROBE=no-runtime\\n"; exit 0 ;;', ' "sandbox ssh-config") for sandbox_ref in "$@"; do :; done; printf "Host openshell-%s\\n HostName 127.0.0.1\\n User sandbox\\n" "$sandbox_ref"; exit 0 ;;', - ` "sandbox create") touch ${JSON.stringify(cloneReadyMarker)} ${JSON.stringify(cloneRunningMarker)}; printf "created clone-1\\n"; exit 0 ;;`, + ` "sandbox create") ${markCloneReady}; touch ${JSON.stringify(cloneRunningMarker)}; printf "created clone-1\\n"; exit 0 ;;`, ` "forward list") printf "SANDBOX BIND PORT PID STATUS\\nclone-1 ${dashboardBind} ${String(dashboardPort)} 4242 running\\n"; exit 0 ;;`, ' "forward stop") exit 1 ;;', "esac", @@ -252,13 +256,17 @@ function makeVmRestoreToEnv( ]); const remoteOpenClawJson = path.join(home, "remote-openclaw.json"); + const snapshotRestoreMarker = path.join(home, "snapshot-restore-attempted"); fs.writeFileSync(remoteOpenClawJson, JSON.stringify({ gateway: { auth: { token: "fresh" } } })); writeExecutable(path.join(localBin, "ssh"), [ `REMOTE_OPENCLAW_JSON=${JSON.stringify(remoteOpenClawJson)}`, + `SNAPSHOT_RESTORE_MARKER=${JSON.stringify(snapshotRestoreMarker)}`, 'cmd=""; for arg do cmd="$arg"; done', 'if printf "%s" "$cmd" | grep -q "openclaw.json"; then', ' if printf "%s" "$cmd" | grep -q "cat --"; then cat "$REMOTE_OPENCLAW_JSON"; exit 0; fi', + ' touch "$SNAPSHOT_RESTORE_MARKER"', ' if printf "%s" "$cmd" | grep -q ".nemoclaw-restore"; then cat > "$REMOTE_OPENCLAW_JSON"; exit 0; fi', + ' exit 92', "fi", "exit 0", ]); @@ -307,6 +315,7 @@ function makeVmRestoreToEnv( return { HOME: home, NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0", + NEMOCLAW_TEST_SNAPSHOT_RESTORE_MARKER: snapshotRestoreMarker, PATH: `${localBin}:${process.env.PATH ?? ""}`, }; } @@ -338,7 +347,7 @@ describe("snapshot VM-driver gateway guard", () => { // `snapshot restore --to ` on VM driver must use the registered // imageTag, not the legacy `docker exec ... kubectl` probe. - it("snapshot restore --to records a fresh clone lifecycle identity before pairing verification (#8942)", () => { + it("snapshot restore --to records a new clone lifecycle generation and live identity (#8942)", () => { const env = makeVmRestoreToEnv("nemoclaw-snap-vm-gw-restore-to-", { imageTag: "openshell/sandbox-from:fast-path-test", lifecycleGeneration: "source-generation", @@ -382,11 +391,59 @@ describe("snapshot VM-driver gateway guard", () => { const r = runCli("alpha snapshot restore baseline --to clone-1", env); expect(r.code, r.out).toBe(1); - expect(r.out).toContain("valid live identity"); + expect(r.out).toContain( + "could not verify the same valid Ready identity from its owning gateway before registration", + ); + expect(r.out).toContain("Snapshot state was not restored and the clone was not registered."); + expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'"); + const registryState = JSON.parse( + fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"), + ); + expect(registryState.sandboxes["clone-1"]).toBeUndefined(); + }, 15000); + + it("snapshot restore --to reports owner-scoped recovery when a created clone is not Ready (#8942)", () => { + const env = makeVmRestoreToEnv( + "nemoclaw-snap-vm-gw-restore-to-not-ready-", + { imageTag: "openshell/sandbox-from:fast-path-test" }, + "fixture-clone-1", + false, + ); + + const r = runCli("alpha snapshot restore baseline --to clone-1", env); + expect(r.code, r.out).toBe(1); + expect(r.out).toContain( + "could not verify the same valid Ready identity from its owning gateway before registration", + ); + expect(r.out).toContain("Snapshot state was not restored and the clone was not registered."); + expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'"); + const registryState = JSON.parse( + fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"), + ); + expect(registryState.sandboxes["clone-1"]).toBeUndefined(); + }, 15000); + + it("snapshot restore --to reports owner-scoped recovery when clone identity changes before registration (#8942)", () => { + const env = makeVmRestoreToEnv( + "nemoclaw-snap-vm-gw-restore-to-identity-drift-", + { imageTag: "openshell/sandbox-from:fast-path-test" }, + "fixture-clone-1", + true, + "fixture-clone-2", + ); + + const r = runCli("alpha snapshot restore baseline --to clone-1", env); + expect(r.code, r.out).toBe(1); + expect(r.out).toContain( + "could not verify the same valid Ready identity from its owning gateway before registration", + ); + expect(r.out).toContain("Snapshot state was not restored and the clone was not registered."); + expect(r.out).toContain("openshell sandbox delete -g 'nemoclaw' 'clone-1'"); const registryState = JSON.parse( fs.readFileSync(path.join(env.HOME, ".nemoclaw", "sandboxes.json"), "utf8"), ); expect(registryState.sandboxes["clone-1"]).toBeUndefined(); + expect(fs.existsSync(env.NEMOCLAW_TEST_SNAPSHOT_RESTORE_MARKER)).toBe(false); }, 15000); it("snapshot restore --to fails closed for VM-driver entries missing imageTag", () => {