feat(runtime): implement durable Docker state mutation

agents/hermes/Dockerfile

@@ -120,7 +120,11 @@ COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-
 COPY agents/hermes/cron-restore-control.py /usr/local/lib/nemoclaw/hermes-cron-restore-control.py
 COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.99.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.99.json
 COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py
+COPY scripts/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py
+COPY scripts/runtime-state-mutation-startup-gate.py /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py
+COPY scripts/runtime_state_mutation_hermes_publisher.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py
 COPY agents/hermes/state-lock-plan.json /usr/local/share/nemoclaw/state-lock-plan.json
+COPY agents/hermes/runtime-state-mutation-publisher-v1.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json
 COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/
 
 FROM scratch AS hermes-wrapper-payload
@@ -363,7 +367,10 @@ RUN discovery_contract="$(node /usr/local/lib/nemoclaw/mcp-tool-discovery-runtim
     && test -f /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \
     && test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \
     && test "$(stat -c '%u:%g:%a' /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh)" = '0:0:444' \
-    && install -d -o root -g root -m 0755 /run/nemoclaw
+    && install -d -o root -g root -m 0755 /run/nemoclaw /var/lib/nemoclaw \
+    && install -d -o root -g root -m 0711 \
+        /run/nemoclaw/runtime-state-mutation-startup \
+        /var/lib/nemoclaw/runtime-state-mutation
 
 # Ensure sandbox user can read blueprint files copied as root
 RUN chmod -R a+rX /opt/nemoclaw-blueprint/
@@ -378,12 +385,14 @@ RUN chmod -R a+rX /opt/nemoclaw-blueprint/
 # minimum supported Hermes sandbox base tag guarantees those artifacts and
 # test/sandbox-rlimit-hooks.test.ts covers that base.
 RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-managed-startup-hold /usr/local/bin/nemoclaw-managed-bootstrap /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py /usr/local/lib/nemoclaw/patch-hermes-sqlite-temp-store.py /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \
-    && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/hermes-cron-restore-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.99.json \
+    && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/hermes-cron-restore-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.99.json \
     && chmod 700 /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/hermes-cron-restore-control.py \
-    && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \
-    && chmod 444 /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/managed_policy.py \
+    && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/lib/nemoclaw/managed-gateway-control.py \
+    && chmod 555 /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py \
+    && chmod 444 /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/managed_policy.py \
     && chmod 444 /usr/local/lib/nemoclaw/patch-hermes-langfuse-credentials.mts \
     && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.99.json \
+    && /opt/hermes/.venv/bin/python3 -I -c 'import runpy, yaml; assert yaml.safe_load("ready: true")["ready"] is True; runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", run_name="nemoclaw_runtime_state_mutation_control_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py", run_name="nemoclaw_runtime_state_mutation_publisher_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py", run_name="nemoclaw_runtime_state_mutation_gate_probe"); runpy.run_path("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", run_name="nemoclaw_runtime_config_guard_probe")' \
     && if [ -d /usr/local/lib/nemoclaw/preloads ]; then \
         chown -R 0:0 /usr/local/lib/nemoclaw/preloads \
         && find /usr/local/lib/nemoclaw/preloads -type f -exec chmod 444 {} + \
@@ -1206,6 +1215,12 @@ RUN check_metadata() { \
     && check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root 444' \
     && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root 700' \
     && check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root 444' \
+    && check_metadata /usr/local/lib/nemoclaw/runtime-state-mutation-control.py 'root:root 500' \
+    && check_metadata /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py 'root:root 555' \
+    && check_metadata /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py 'root:root 500' \
+    && check_metadata /var/lib/nemoclaw/runtime-state-mutation 'root:root 711' \
+    && check_metadata /run/nemoclaw/runtime-state-mutation-startup 'root:root 711' \
+    && check_metadata /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json 'root:root 444' \
     && check_metadata /usr/local/lib/nemoclaw/hermes-cron-restore-control.py 'root:root 700' \
     && check_metadata /usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js 'root:root 444' \
     && check_metadata /usr/local/lib/nemoclaw/hermes-wrapper.py 'root:root 755' \

agents/hermes/runtime-state-mutation-publisher-v1.json

@@ -0,0 +1 @@
+{"schemaVersion":1,"protocol":"nemoclaw-runtime-state-mutation-publisher-v1","agent":"hermes","providerId":"docker","stateRoot":"/sandbox/.hermes","planSchemaVersion":2,"entrypoint":"/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py"}

agents/hermes/start.sh

@@ -19,6 +19,105 @@ set -euo pipefail
 
 # SECURITY: Lock down PATH before resolving or sourcing root startup helpers.
 export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV=("$@")
+
+# The provider gate is a fixed root-owned file beneath a search-only directory
+# outside /sandbox, so the sandbox identity cannot rename either the gate or
+# its parent.  Run the immutable validator as this shell's direct child before
+# sourcing helpers or reading mutable state; its permit is bound to this exact
+# process identity.  Invalid or uninspectable gate state is a hold, never an
+# availability-to-integrity downgrade.
+readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON="/opt/hermes/.venv/bin/python3"
+readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER="/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py"
+readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV="/usr/bin/setpriv"
+
+if [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" ] \
+  || [ ! -f "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \
+  || [ -L "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \
+  || { [ "$EUID" -eq 0 ] && [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" ]; }; then
+  printf '%s\n' '[SECURITY] Required runtime state mutation startup gate is unavailable.' >&2
+  exit 1
+fi
+
+nemoclaw_runtime_state_mutation_gate() {
+  local action="$1"
+  if [ "$EUID" -eq 0 ]; then
+    "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \
+      --reuid=sandbox --regid=sandbox --init-groups -- \
+      "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \
+      "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" "$action" >/dev/null
+    return
+  fi
+  "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \
+    "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" "$action" >/dev/null
+}
+
+nemoclaw_runtime_state_mutation_retry_exec() {
+  local status
+  if nemoclaw_runtime_state_mutation_gate restart; then
+    status=0
+  else
+    status=$?
+  fi
+  if [ "$status" -eq 12 ]; then
+    exec /usr/local/bin/nemoclaw-start \
+      "${NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV[@]}"
+  fi
+  printf '%s\n' '[SECURITY] Runtime state mutation retry was not authenticated; holding startup.' >&2
+  kill -STOP "$$"
+}
+trap nemoclaw_runtime_state_mutation_retry_exec USR2
+
+while :; do
+  if nemoclaw_runtime_state_mutation_gate admit; then
+    break
+  else
+    _nemoclaw_runtime_state_mutation_gate_status=$?
+  fi
+  case "$_nemoclaw_runtime_state_mutation_gate_status" in
+    10) break ;;
+    75)
+      printf '%s\n' '[SECURITY] Hermes startup held by an active runtime state mutation.' >&2
+      /bin/sleep 1 || true
+      ;;
+    *)
+      printf '%s\n' '[SECURITY] Runtime state mutation startup gate failed.' >&2
+      exit 1
+      ;;
+  esac
+done
+unset _nemoclaw_runtime_state_mutation_gate_status
+
+# Publish a candidate only after the complete gateway topology is healthy.
+# The shell then stops itself until the root controller has independently
+# authenticated the candidate, frozen the exact process tree, and published a
+# release receipt.  Calling this with no active mutation is a cheap no-op.
+nemoclaw_runtime_state_mutation_checkpoint() {
+  local status
+  if nemoclaw_runtime_state_mutation_gate checkpoint; then
+    return 0
+  else
+    status=$?
+  fi
+  if [ "$status" -ne 11 ]; then
+    printf '%s\n' '[SECURITY] Runtime state mutation startup checkpoint was refused; holding startup.' >&2
+    kill -STOP "$$"
+    return 1
+  fi
+  kill -STOP "$$"
+  if nemoclaw_runtime_state_mutation_gate resume; then
+    return 0
+  else
+    status=$?
+  fi
+  if [ "$status" -eq 12 ]; then
+    exec /usr/local/bin/nemoclaw-start \
+      "${NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV[@]}"
+  fi
+  printf '%s\n' '[SECURITY] Runtime state mutation release receipt was not authenticated; holding startup.' >&2
+  kill -STOP "$$"
+  return 1
+}
 
 # managed-entrypoint-env-wrapper begin
 _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh"
@@ -127,6 +226,7 @@ exec > >(tee -a "$_START_LOG") 2> >(tee -a "$_START_LOG" >&2)
 drop_capabilities /usr/local/bin/nemoclaw-start "$@"
 
 NEMOCLAW_CMD=("$@")
+NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV=("${NEMOCLAW_CMD[@]}")
 
 _chat_ui_url_port() {
   [ -n "${CHAT_UI_URL:-}" ] || return 1
@@ -2862,6 +2962,11 @@ handle_hermes_gateway_control_request() {
     return 1
   fi
   refresh_hermes_supervised_child_pids
+  nemoclaw_runtime_state_mutation_checkpoint || {
+    stop_hermes_gateway_fail_closed
+    gateway_control_fail internal "$old_pid"
+    return 1
+  }
   gateway_control_complete ok "$old_pid" "$GATEWAY_PID"
 }
 
@@ -3090,6 +3195,7 @@ recover_hermes_gateway_current_user() {
             return 1
           fi
           refresh_hermes_supervised_child_pids
+          nemoclaw_runtime_state_mutation_checkpoint || return 1
           return 0
         fi
         echo "[gateway] Hermes auxiliary repair failed; retrying while the exact gateway remains healthy" >&2
@@ -3182,6 +3288,7 @@ bootstrap_hermes_gateway_current_user() {
       return 1
     fi
     refresh_hermes_supervised_child_pids
+    nemoclaw_runtime_state_mutation_checkpoint || return 1
     return 0
   fi
 
@@ -3197,6 +3304,7 @@ bootstrap_hermes_gateway_current_user() {
   sleep 2 || true
   recover_hermes_gateway_current_user || return 1
   refresh_hermes_supervised_child_pids
+  nemoclaw_runtime_state_mutation_checkpoint || return 1
 }
 
 # ── Main ─────────────────────────────────────────────────────────
@@ -3325,6 +3433,7 @@ if ! "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" publish-startup-ready
   echo "[gateway-control] failed to publish Hermes startup readiness" >&2
   exit 1
 fi
+nemoclaw_runtime_state_mutation_checkpoint || exit 1
 print_dashboard_urls
 
 # PID 1 remains alive even when Hermes stops its gateway. Host recovery uses