fix(rebuild): restart the Hermes gateway after state restore

agents/hermes/Dockerfile

@@ -569,7 +569,7 @@ ARG NEMOCLAW_HERMES_CLI_ADAPTER_SHA256=989edf54a8c09c6efb348600a8aa2f264c0b71408
 ARG NEMOCLAW_HERMES_CLI_ADAPTER_VALIDATOR_SHA256=db4046e79e513eab67b069a8eda20167b8b65529cf26842531d2ad673c670330
 ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=822c7e63d068c5d09f3291350771c1a42c9686f51bfa9bc9a1f41fbe15d163b1
 ARG NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256=a1e6b1c53ab297569abb87c29d15c294d729e46005bfd022136b4c447a791819
-ARG NEMOCLAW_HERMES_CRON_RESTORE_CONTROLLER_SHA256=a674de12d1c30d907491aa7c7b5d40711b077c5b2eca69ca58c5c36e8231e4c8
+ARG NEMOCLAW_HERMES_CRON_RESTORE_CONTROLLER_SHA256=f24a4dc187428530bfec2f95140fbe2eff59d9dd78d8edac352bcdc6b6a82586
 # hadolint ignore=DL4006
 RUN printf '%s  %s\n' \
         "$NEMOCLAW_HERMES_VALIDATOR_SHA256" /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \

agents/hermes/cron-restore-control.py

@@ -3,10 +3,12 @@
 """Control Hermes cron dispatch while NemoClaw restores durable state.
 
 Cron restore control is the rebuild-time gate that keeps dispatch disabled until
-backed-up scripts and job definitions are valid and the gateway identity is
-unchanged. The gateway identity is the (PID, start_time) tuple pinned across
-begin, validate, and release. A drain token is the client-side secret proving
-ownership of the server-side persisted drain marker.
+backed-up scripts and job definitions are valid and the replacement gateway is
+ready. The initial gateway identity is pinned across begin and validate. The
+replacement identity is observed around managed health verification, and the
+complete action requires that same live identity before releasing the gate. A
+drain token is the client-side secret proving ownership of the server-side
+persisted drain marker.
 """
 
 from __future__ import annotations
@@ -31,6 +33,9 @@
 CONTROL_LOCK_PATH = Path("/run/nemoclaw/hermes-cron-restore-control.lock")
 CONTROL_MARKER_NAME = "hermes-cron-restore-drain.json"
 RECEIPT_PREFIX = "NEMOCLAW_HERMES_CRON_RESTORE_V1:"
+CONTROL_ERROR_PREFIX = "NEMOCLAW_HERMES_CRON_RESTORE_ERROR_V1:"
+CONTROL_ERROR_CODE = "control-failure"
+DRAIN_MARKER_ROLLBACK_FAILED_CODE = "drain-marker-rollback-failed"
 BEGIN_TIMEOUT_SECONDS = 60.0
 RELEASE_TIMEOUT_SECONDS = 15.0
 POLL_SECONDS = 0.1
@@ -43,6 +48,24 @@
 class ControlError(RuntimeError):
     """Expected fail-closed control or validation error."""
 
+    def __init__(self, message: str, *, code: str = CONTROL_ERROR_CODE) -> None:
+        super().__init__(message)
+        self.code = code
+
+
+def _emit_control_error(error: ControlError) -> None:
+    """Write the stable control signal after the existing human-readable error."""
+    print(f"HERMES_CRON_RESTORE_ERROR: {error}", file=sys.stderr)
+    print(
+        CONTROL_ERROR_PREFIX
+        + json.dumps(
+            {"code": error.code, "message": str(error)},
+            separators=(",", ":"),
+            sort_keys=True,
+        ),
+        file=sys.stderr,
+    )
+
 
 def _marker_path() -> Path:
     return NEMOCLAW_HOME / CONTROL_MARKER_NAME
@@ -480,6 +503,7 @@ def _complete_release(
     drain_token: str,
     **fields: Any,
 ) -> None:
+    _require_drained_idle(status_module, pid, start_time)
     _remove_owned_drain(drain_token)
     try:
         payload, operator_drain_active, disposition = _wait_for_release_disposition(
@@ -493,7 +517,8 @@ def _complete_release(
             _write_owned_drain(drain_token)
         except ControlError as rollback_error:
             raise ControlError(
-                "Hermes cron restore drain release failed and its marker could not be restored"
+                "Hermes cron restore drain release failed and its marker could not be restored",
+                code=DRAIN_MARKER_ROLLBACK_FAILED_CODE,
             ) from rollback_error
         if isinstance(release_error, ControlError):
             raise release_error
@@ -554,18 +579,63 @@ def validate_restore(pid: int, start_time: int, drain_token: str) -> None:
         )
 
 
-def release_drain(pid: int, start_time: int, drain_token: str) -> None:
+def observe_replacement(pid: int, start_time: int, drain_token: str) -> None:
     with _control_lock():
         drain_control, status_module = _load_gateway_modules()
         _require_owned_drain(drain_token)
-        _require_drained_idle(status_module, pid, start_time)
+        _, replacement_pid, replacement_start_time = _gateway_identity(status_module)
+        if replacement_pid == pid and replacement_start_time == start_time:
+            raise ControlError("Hermes gateway identity did not change during cron restore")
+        payload = _wait_for_state(
+            status_module,
+            pid=replacement_pid,
+            start_time=replacement_start_time,
+            state="draining",
+            require_idle=True,
+            timeout_seconds=BEGIN_TIMEOUT_SECONDS,
+        )
+        _receipt(
+            "observe",
+            replacement_pid,
+            replacement_start_time,
+            drain_token,
+            active_agents=status_module.parse_active_agents(
+                payload.get("active_agents")
+            ),
+            disposition="replacement-observed",
+            operator_drain_active=_operator_drain_active(drain_control),
+        )
+
+
+def complete_replacement(
+    pid: int,
+    start_time: int,
+    replacement_pid: int,
+    replacement_start_time: int,
+    drain_token: str,
+) -> None:
+    with _control_lock():
+        drain_control, status_module = _load_gateway_modules()
+        _require_owned_drain(drain_token)
+        if replacement_pid == pid and replacement_start_time == start_time:
+            raise ControlError("Hermes gateway identity did not change during cron restore")
+        _wait_for_state(
+            status_module,
+            pid=replacement_pid,
+            start_time=replacement_start_time,
+            state="draining",
+            require_idle=True,
+            timeout_seconds=BEGIN_TIMEOUT_SECONDS,
+        )
+        counts = validate_cron_tree()
         _complete_release(
-            "release",
+            "complete",
             drain_control,
             status_module,
-            pid=pid,
-            start_time=start_time,
+            pid=replacement_pid,
+            start_time=replacement_start_time,
             drain_token=drain_token,
+            **counts,
         )
 
 
@@ -614,11 +684,16 @@ def _parser() -> argparse.ArgumentParser:
     subparsers = parser.add_subparsers(dest="action", required=True)
     subparsers.add_parser("begin")
     subparsers.add_parser("recover")
-    for action in ("validate", "release"):
+    for action in ("validate", "observe", "complete"):
         subparser = subparsers.add_parser(action)
         subparser.add_argument("--pid", required=True, type=int)
         subparser.add_argument("--start-time", required=True, type=int)
         subparser.add_argument("--drain-token", required=True)
+        if action == "complete":
+            subparser.add_argument("--replacement-pid", required=True, type=int)
+            subparser.add_argument(
+                "--replacement-start-time", required=True, type=int
+            )
     tree = subparsers.add_parser("validate-tree")
     tree.add_argument("--home", required=True, type=Path)
     tree.add_argument("--sandbox-home", required=True, type=Path)
@@ -634,13 +709,21 @@ def main() -> int:
             recover_drain()
         elif args.action == "validate":
             validate_restore(args.pid, args.start_time, args.drain_token)
-        elif args.action == "release":
-            release_drain(args.pid, args.start_time, args.drain_token)
+        elif args.action == "observe":
+            observe_replacement(args.pid, args.start_time, args.drain_token)
+        elif args.action == "complete":
+            complete_replacement(
+                args.pid,
+                args.start_time,
+                args.replacement_pid,
+                args.replacement_start_time,
+                args.drain_token,
+            )
         else:
             counts = validate_cron_tree(args.home, args.sandbox_home)
             print(json.dumps(counts, separators=(",", ":"), sort_keys=True))
     except ControlError as error:
-        print(f"HERMES_CRON_RESTORE_ERROR: {error}", file=sys.stderr)
+        _emit_control_error(error)
         return 1
     return 0
 

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -221,9 +221,10 @@ $$nemoclaw <sandbox-name> gateway-token --quiet
 ```
 
 Before post-restore repairs, NemoClaw verifies that the recreated sandbox still identifies as Hermes and exits nonzero if its identity does not match the rebuild target.
-After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then re-proves or recovers gateway health and performs final MCP reconciliation.
+After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then restarts the Hermes gateway and verifies or recovers its health before performing final MCP reconciliation.
+The gateway starts during recreation and reads its durable state before the restore replaces it, so the restart is what binds the running gateway to the restored state.
 `rebuild` exits nonzero instead of reporting success when it cannot verify final gateway health or managed MCP state.
-Follow the printed recovery guidance, using `$$nemoclaw <sandbox-name> recover` for gateway health and `$$nemoclaw <sandbox-name> mcp restart` for incomplete managed MCP restoration.
+Follow the printed recovery guidance, using `$$nemoclaw <sandbox-name> gateway restart` first for gateway health, `$$nemoclaw <sandbox-name> recover` when the restart does not restore verified health, and `$$nemoclaw <sandbox-name> mcp restart` for incomplete managed MCP restoration.
 
 When the rebuild backup contains active Hermes cron jobs that reference scripts, NemoClaw validates those script references before it deletes the existing sandbox.
 The check covers the default profile and named profiles.
@@ -234,12 +235,16 @@ If this validation fails, the rebuild keeps the existing sandbox and reports the
 After NemoClaw creates the replacement, it acquires an independent root-owned gate that blocks new Hermes turns and cron dispatch.
 The gate remains active across gateway and container restarts in the replacement sandbox.
 NemoClaw waits for active agent work to finish before restoring state.
-It validates the restored jobs and scripts against the same running gateway before it clears its gate.
+It validates the restored jobs and scripts before the gateway replacement, then keeps dispatch blocked while it restarts and verifies that replacement.
+It records the replacement process identity around managed health verification and clears the gate only if that same live process completes the final cron validation.
 If an operator already drained the gateway, NemoClaw clears only its gate and leaves the operator drain active.
-If state restore or cron validation fails after gate acquisition, the command exits nonzero, preserves the backup, and retains the NemoClaw gate.
+If state restore, managed MCP restoration, gateway replacement, or cron validation fails after gate acquisition, the command exits nonzero and preserves the backup.
+Those failures retain the NemoClaw gate unless the output explicitly reports that release rollback could not restore its marker.
+In that exceptional case, do not assume dispatch is blocked; run `$$nemoclaw <sandbox-name> recover` immediately.
 Failures before gate acquisition do not create a new gate.
 Do not manually remove the root-owned cron restore marker because removal bypasses restored cron validation.
-After you correct the reported restore problem, run `$$nemoclaw <sandbox-name> recover` to validate the restored cron tree and clear the NemoClaw gate.
+If managed MCP restoration failed, correct the reported cause and run `$$nemoclaw <sandbox-name> mcp restart` first.
+Then run `$$nemoclaw <sandbox-name> recover` to repair and probe the gateway, validate the restored cron tree, and clear the NemoClaw gate.
 </AgentOnly>
 
 <AgentOnly variant="deepagents">