diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8f9515af132..7394b419845 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -4,13 +4,13 @@ "defaultMax": 20, "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, - "src/lib/actions/sandbox/process-recovery.ts": 26, + "src/lib/actions/sandbox/process-recovery.ts": 27, "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 53, - "src/lib/adapters/openshell/timeouts.ts": 37, - "src/lib/agent/defs.ts": 32, + "src/lib/adapters/openshell/runtime.ts": 54, + "src/lib/adapters/openshell/timeouts.ts": 38, + "src/lib/agent/defs.ts": 33, "src/lib/cli/branding.ts": 86, "src/lib/cli/nemoclaw-oclif-command.ts": 106, "src/lib/cli/terminal-style.ts": 43, @@ -20,15 +20,15 @@ "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, "src/lib/credentials/store.ts": 46, - "src/lib/inference/config.ts": 29, + "src/lib/inference/config.ts": 30, "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, - "src/lib/onboard/gateway-binding.ts": 50, + "src/lib/onboard/gateway-binding.ts": 51, "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 52, "src/lib/state/onboard-session.ts": 36, - "src/lib/state/registry.ts": 100, - "src/lib/state/state-root.ts": 20, + "src/lib/state/registry.ts": 101, + "src/lib/state/state-root.ts": 21, "src/lib/subprocess-env.ts": 24, "src/lib/validation.ts": 25 } @@ -37,7 +37,7 @@ "defaultMax": 20, "maxByFile": { "src/lib/actions/inference-set.ts": 32, - "src/lib/actions/sandbox/connect.ts": 38, + "src/lib/actions/sandbox/connect.ts": 39, "src/lib/actions/sandbox/destroy.ts": 29, "src/lib/actions/sandbox/doctor.ts": 30, "src/lib/actions/sandbox/status-snapshot.ts": 20, @@ -58,8 +58,8 @@ "maxRootFiles": { "src/lib/onboard": 308, "src/lib/actions": 19, - "src/lib/actions/sandbox": 182, - "src/lib/state": 37, + "src/lib/actions/sandbox": 183, + "src/lib/state": 38, "src/lib/inference": 63, "scripts": 47 } diff --git a/docs/get-started/quickstart-hermes.mdx b/docs/get-started/quickstart-hermes.mdx index 0eb4475f810..3cad1280af7 100644 --- a/docs/get-started/quickstart-hermes.mdx +++ b/docs/get-started/quickstart-hermes.mdx @@ -83,7 +83,10 @@ Review the [Prerequisites](prerequisites) before you begin. nemohermes launch my-hermes ``` - `nemohermes launch` runs the same preflight as `connect`, then starts `hermes` in your terminal. + `nemohermes launch` runs the complete preflight or, on Linux, validates an existing fixed 24-hour launch-readiness lease before it starts `hermes` in your terminal. + On macOS, `launch` runs the complete preflight every time and does not publish a launch-readiness lease. + On Linux, if the recorded configuration or live runtime has changed, NemoClaw durably invalidates prior evidence before it runs the complete preflight. + If old evidence might remain acceptable and cannot be invalidated, `launch` stops before preflight or recovery and tells you to repair the secure OS per-user runtime authority or NemoClaw state permissions. To open a sandbox shell first and start Hermes yourself, run these commands instead. ```bash diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 3b375d4c233..94d392e26bb 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -80,7 +80,10 @@ Review the [Prerequisites](prerequisites) before you begin. nemo-deepagents launch my-deepagents ``` - `nemo-deepagents launch` runs the same preflight as `connect`, then starts `dcode` in your terminal. + `nemo-deepagents launch` runs the complete preflight or, on Linux, validates an existing fixed 24-hour launch-readiness lease before it starts `dcode` in your terminal. + On macOS, `launch` runs the complete preflight every time and does not publish a launch-readiness lease. + On Linux, if the recorded configuration or live runtime has changed, NemoClaw durably invalidates prior evidence before it runs the complete preflight. + If old evidence might remain acceptable and cannot be invalidated, `launch` stops before preflight or recovery and tells you to repair the secure OS per-user runtime authority or NemoClaw state permissions. To open a sandbox shell first and start `dcode` yourself, run these commands instead. ```bash diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index dffc904b157..7927f6ab82d 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -81,7 +81,10 @@ Review the [Prerequisites](prerequisites) before you begin. nemoclaw launch my-assistant ``` - `nemoclaw launch` runs the same preflight as `connect`, then starts `openclaw tui` in your terminal. + `nemoclaw launch` runs the complete preflight or, on Linux, validates an existing fixed 24-hour launch-readiness lease before it starts `openclaw tui` in your terminal. + On macOS, `launch` runs the complete preflight every time and does not publish a launch-readiness lease. + On Linux, if the recorded configuration or live runtime has changed, NemoClaw durably invalidates prior evidence before it runs the complete preflight. + If old evidence might remain acceptable and cannot be invalidated, `launch` stops before preflight or recovery and tells you to repair the secure OS per-user runtime authority or NemoClaw state permissions. To open a sandbox shell first and start the TUI yourself, run these commands instead. ```bash diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index e1de7dba6f2..f267e4c41f3 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -138,6 +138,47 @@ Use `nemo-deepagents status`, `logs`, `connect`, and `rebuild` fo If the terminal runtime reports degraded health, rebuild the sandbox instead of using `recover` or `gateway restart`. +### Understand Launch Readiness Leases + +A successful complete preflight for `$$nemoclaw launch ` can publish a credential-free launch-readiness lease on Linux. +Linux infrastructure can publish the same evidence with `$$nemoclaw connect --probe-only`. +The lease has a fixed 24-hour lifetime that repeated launches do not extend. +Leaving the agent with `/exit` does not revoke it, and users do not refresh it manually. + +During the lease, `launch` still verifies the owning OpenShell gateway, exact live sandbox identity, registry and agent configuration, effective policy, inference route, required forwards, and semantic runtime health. +Configured inference must return HTTP 2xx from the semantic `inference.local` probe, which is stricter than the HTTP 200–499 reachability diagnostic used by ordinary `connect`. +When those checks pass, it can skip duplicate recovery, readiness polling, and inference-route repair. +The lease is not a health guarantee or repair authority. + +For missing, unsafe, malformed, expired, mismatched, changed, or unhealthy evidence, NemoClaw fences any prior acceptable evidence before it runs the complete preflight. +Ordinary launch continues only when NemoClaw proves that no old authority or evidence can exist, or durably rotates the runtime epoch. +If an old epoch might exist and cannot be durably rotated, `launch` and `connect --probe-only` stop before complete preflight or recovery. +Their redacted guidance asks you to repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry. +If NemoClaw securely proves that both the authority and receipt are absent but cannot create new authority, ordinary `launch` can run the complete preflight without optimization; `connect --probe-only` exits nonzero because it could not publish evidence. +If that preflight succeeds before the lease expires, replacement evidence keeps the original start and expiry time. +After expiry, a successful complete preflight starts a new 24-hour lease only when publication succeeds. + +Before the first mutation in the complete preflight, the producer revalidates its sandbox-global runtime epoch under the sandbox lifecycle lock followed by the owning gateway lock. +It holds both locks through all mutations in the complete preflight, final state capture, and publication. +A stale producer makes no changes and re-inspects the newer lease. + +If unsafe or malformed authority history makes the prior lease timeline untrustworthy, NemoClaw durably invalidates the old epoch and starts one conservative 24-hour quarantine. +Publication remains disabled until both wall time and monotonic uptime span the full quarantine. +Repeated attempts do not extend it. +After it elapses, the next successful complete preflight can publish a new fixed 24-hour lease. + +Lease acceptance and publication are currently Linux-only and require a secure, independently writable OS per-user runtime authority under `/run/user/`. +It never uses caller-provided environment variables to select this authority. + +On macOS, `launch` runs the complete preflight every time and does not publish a launch-readiness lease. +`connect --probe-only` also runs the complete preflight, including recovery and probes, but exits nonzero because it cannot publish authoritative launch-readiness evidence. +The publication-failure diagnostic is redacted and does not print filesystem paths or environment values. + +Infrastructure must run `connect --probe-only` as the same final numeric user that later runs `launch`. +Run it after the final durable home and state volume is mounted and after policy and network provisioning is complete. +NemoClaw rejects evidence after a bound sandbox, configuration, policy, or network identity changes. +Deployment ordering remains responsible for external changes that OpenShell and NemoClaw cannot observe. + ### Host OpenShell Gateway Versus In-Sandbox Agent Recovery diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 4ebd371a2ed..3c1529b659f 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1013,9 +1013,44 @@ $$nemoclaw use --json Connect to a sandbox and start its agent in one host-side command. Use it instead of running `$$nemoclaw connect` and then typing the agent command inside the sandbox. -`launch` runs the same preflight as [`$$nemoclaw connect`](#$$nemoclaw-name-connect), including the readiness wait, in-sandbox agent process recovery, and inference-route reconciliation. -It runs every check in that shared preflight. -After the preflight, `launch` starts the sandbox's agent in your terminal instead of opening a sandbox shell. +`launch` runs the complete preflight from [`$$nemoclaw connect`](#$$nemoclaw-name-connect) when no launch-readiness lease is usable. +That path includes the readiness wait, in-sandbox agent process recovery, and inference-route reconciliation. +A successful complete preflight can publish a credential-free launch-readiness lease with a fixed 24-hour lifetime on Linux. +Lease acceptance and publication are currently Linux-only and require a secure, independently writable OS per-user runtime authority under `/run/user/`. +It never uses caller-provided environment variables to select this authority. +On macOS, `launch` runs the complete preflight every time and does not publish a launch-readiness lease. + +During that lease, another `launch` still verifies these conditions: + +- The owning OpenShell gateway reports the exact sandbox identity in the `Ready` or `Running` state. +- The sandbox registry, agent manifest, interactive command, policy intent, and effective parsed OpenShell network policy match the recorded identity. +- The recorded inference selection matches the live route, and `inference.local` returns HTTP 2xx from its semantic probe when inference is configured. This is stricter than the HTTP 200–499 reachability diagnostic used by ordinary `connect`. +- The agent runtime and its required host-side forwards pass their semantic health checks. + +After these checks pass, `launch` can skip duplicate recovery, readiness polling, and inference-route repair. +The lease does not replace a health check or authorize repair. +For missing, expired, malformed, inaccessible, mismatched, or unhealthy evidence, NemoClaw fences any prior acceptable evidence before it runs the complete preflight. +Ordinary launch continues only when NemoClaw proves that no old authority or evidence can exist, or durably rotates the runtime epoch. +If an old epoch might exist and cannot be durably rotated, `launch` stops before complete preflight or recovery. +Its redacted guidance asks you to repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry. +A failed live check never becomes a successful launch because a lease exists. + +Immediately before the first mutation in the complete preflight, the producer revalidates its sandbox-global runtime epoch while holding the sandbox lifecycle lock followed by the owning gateway lock. +It holds both locks through all mutations in the complete preflight, final state capture, and publication. +If another producer has replaced the epoch, the stale producer makes no changes and re-inspects the newer lease. + +The 24-hour lifetime does not extend when you launch repeatedly. +Exiting the agent with `/exit` does not revoke the lease. +If state changes before expiry, NemoClaw fences the old evidence and runs the complete preflight. +A successful preflight in that interval keeps the original start and expiry time. +After expiry, a successful complete preflight starts a new 24-hour lease only when publication succeeds. + +If unsafe or malformed authority history makes the prior lease timeline untrustworthy, NemoClaw durably invalidates the old epoch and starts one conservative 24-hour quarantine. +Both wall time and monotonic uptime must span the full quarantine, and publication remains disabled during it. +Repeated attempts do not extend the quarantine. +After it elapses, the next successful complete preflight can publish a new fixed 24-hour lease. +You do not create or refresh this lease manually, and `launch` has no lease-control flags. +After lease validation or the automatic fallback that runs the complete preflight, `launch` starts the sandbox's agent in your terminal instead of opening a sandbox shell. The agent command comes from the sandbox's agent manifest. If the sandbox registry names a non-OpenClaw agent without a readable local agent manifest, `launch` exits before starting an in-sandbox command. @@ -1031,6 +1066,8 @@ $$nemoclaw launch ``` The sandbox name is required, and the command takes no flags. +The sandbox must already exist in the local NemoClaw state. +If it is not registered locally, `launch` exits before it runs an OpenShell command or readiness recovery and reports that the sandbox is not registered in the local NemoClaw state. When the agent exits, you return to the host shell. @@ -1118,11 +1155,24 @@ $$nemoclaw my-assistant connect [--probe-only] $$nemoclaw connect ``` -The `--probe-only` flag waits for the sandbox to become ready, verifies or repairs its in-sandbox agent process and host-side forward, and exits without opening a shell. +On Linux, the `--probe-only` flag is the infrastructure producer for launch-readiness evidence. +It validates a usable lease and exits without duplicate recovery. +Otherwise, it fences prior evidence, waits for the sandbox, verifies or repairs its in-sandbox agent process and host-side forwards, and publishes evidence only after every probe succeeds. It rechecks the sandbox on its recorded OpenShell gateway after the readiness wait and never restarts the shared host gateway. -Use it for health checks and scripted readiness probes. - -Use [`$$nemoclaw launch `](#$$nemoclaw-launch-name) when you want the same preflight followed by the agent instead of a sandbox shell. +If an old runtime epoch might exist and cannot be durably rotated, the command exits nonzero before complete preflight or recovery and gives redacted repair guidance. +A securely absent runtime authority and receipt let ordinary `launch` run the complete preflight without optimization if new authority creation fails, but `connect --probe-only` still exits nonzero because it could not publish launch-readiness evidence. +A runtime failure and a failure to publish evidence for an otherwise healthy runtime also exit nonzero with different diagnostics. + +Infrastructure must run the command as the same final numeric user that later runs `launch`. +Run it only after the final durable home and state volume is mounted and after policy and network provisioning is complete. +On Linux, that user also needs a secure, independently writable OS per-user runtime authority under `/run/user/`. +Do not redirect this authority with caller environment variables. +Do not use a graphical or login-session identifier as the deployment ordering boundary. +On macOS, `connect --probe-only` runs the complete preflight, including recovery and probes, but exits nonzero because it cannot publish authoritative launch-readiness evidence. +The publication-failure diagnostic is redacted and does not print filesystem paths or environment values. +Run it for health checks and scripted readiness probes; users continue to run only `$$nemoclaw launch `. + +Use [`$$nemoclaw launch `](#$$nemoclaw-launch-name) when you want launch-readiness validation, an automatic fallback that runs the complete preflight, and then the agent instead of a sandbox shell. ### `$$nemoclaw exec` diff --git a/scripts/test-launch-readiness-lease.sh b/scripts/test-launch-readiness-lease.sh new file mode 100755 index 00000000000..274f0527d95 --- /dev/null +++ b/scripts/test-launch-readiness-lease.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "Launch-readiness lease acceptance requires Linux and the util-linux PTY driver." >&2 + exit 2 +fi + +if [[ $# -ne 1 || -z "$1" ]]; then + echo "Usage: scripts/test-launch-readiness-lease.sh " >&2 + exit 2 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "$repo_root" + +npm run clean:cli +npm run build:cli +NEMOCLAW_RUN_LIVE_E2E=1 \ + NEMOCLAW_ACCEPTANCE_SANDBOX="$1" \ + npx vitest run --project e2e-live \ + test/e2e/live/launch-readiness-lease-acceptance.test.ts diff --git a/src/commands/launch.ts b/src/commands/launch.ts index ff3a3f19ad4..ce5ebb084b0 100644 --- a/src/commands/launch.ts +++ b/src/commands/launch.ts @@ -10,7 +10,7 @@ export default class LaunchCommand extends NemoClawCommand { static strict = true; static summary = "Connect to a sandbox and start its agent"; static description = - "Run the same preflight as connect, then start the sandbox's agent in one step so an interactive session does not need a second command typed inside the sandbox."; + "Validate a current launch-readiness lease or run the complete connect preflight, then start the sandbox's agent in one step so an interactive session does not need a second command typed inside the sandbox."; static usage = ["launch "]; static examples = ["<%= config.bin %> launch alpha"]; static args = { diff --git a/src/lib/actions/sandbox/auto-pair-approval-connect.test.ts b/src/lib/actions/sandbox/auto-pair-approval-connect.test.ts index 142f6080ec4..6010f975d8c 100644 --- a/src/lib/actions/sandbox/auto-pair-approval-connect.test.ts +++ b/src/lib/actions/sandbox/auto-pair-approval-connect.test.ts @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { runConnectAutoPairApprovalPass } from "./auto-pair-approval"; +import { + runConnectAutoPairApprovalPass, + runSandboxAutoPairApprovalPass, +} from "./auto-pair-approval"; import { CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, CONNECT_AUTO_PAIR_LIST_TIMEOUT_S, @@ -12,10 +15,14 @@ import { } from "./connect-autopair-budget"; describe("connect auto-pair approval pass", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("uses the shared connect approval budget", () => { const runApprovalPass = vi.fn(); - runConnectAutoPairApprovalPass("alpha", runApprovalPass); + runConnectAutoPairApprovalPass("alpha", "nemoclaw-8091", runApprovalPass); expect(runApprovalPass).toHaveBeenCalledWith("alpha", { budget: { @@ -24,6 +31,36 @@ describe("connect auto-pair approval pass", () => { approveTimeoutS: CONNECT_AUTO_PAIR_APPROVE_TIMEOUT_S, timeoutMs: CONNECT_AUTO_PAIR_TIMEOUT_MS, }, + gatewayName: "nemoclaw-8091", }); }); + + it("pins sandbox exec to the owning OpenShell gateway despite ambient gateway drift (#8942)", () => { + vi.stubEnv("OPENSHELL_GATEWAY", "ambient-sibling"); + const spawn = vi.fn((_binary: string, _args: readonly string[]) => ({ + status: 0, + signal: null, + stdout: "__NEMOCLAW_AUTO_PAIR_APPROVED__=0\n", + stderr: "", + })); + + runSandboxAutoPairApprovalPass( + "alpha", + { capture: true, gatewayName: "nemoclaw-8091" }, + { getOpenshellBinary: () => "openshell", spawnSync: spawn as never }, + ); + + expect(process.env.OPENSHELL_GATEWAY).toBe("ambient-sibling"); + expect(spawn.mock.calls[0]?.[1]).toEqual([ + "sandbox", + "exec", + "--name", + "alpha", + "-g", + "nemoclaw-8091", + "--", + "sh", + "-s", + ]); + }); }); diff --git a/src/lib/actions/sandbox/auto-pair-approval.ts b/src/lib/actions/sandbox/auto-pair-approval.ts index 9baae106d11..973319a3a4f 100644 --- a/src/lib/actions/sandbox/auto-pair-approval.ts +++ b/src/lib/actions/sandbox/auto-pair-approval.ts @@ -1066,6 +1066,7 @@ export function runSandboxAutoPairApprovalPass( receipt?: boolean; budget?: AutoPairApprovalBudget; localDeviceOnly?: boolean; + gatewayName?: string; } = {}, execDeps?: AutoPairApprovalExecDeps, ): AutoPairApprovalResult { @@ -1104,7 +1105,16 @@ export function runSandboxAutoPairApprovalPass( // so script growth cannot fail before Python emits its fixed receipt. const result = deps.spawnSync( deps.getOpenshellBinary(), - ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-s"], + [ + "sandbox", + "exec", + "--name", + sandboxName, + ...(options.gatewayName ? ["-g", options.gatewayName] : []), + "--", + "sh", + "-s", + ], { cwd: ROOT, env: process.env, @@ -1140,7 +1150,21 @@ export function runSandboxAutoPairApprovalPass( /** Run the approval pass with the shared connect, probe, and finalization budget. */ export function runConnectAutoPairApprovalPass( sandboxName: string, + gatewayName?: string, runApprovalPass = runSandboxAutoPairApprovalPass, ): void { - runApprovalPass(sandboxName, { budget: CONNECT_AUTO_PAIR_BUDGET }); + let owningGatewayName = gatewayName; + if (!owningGatewayName) { + try { + owningGatewayName = ( + require("./gateway-target") as typeof import("./gateway-target") + ).getSandboxTargetGatewayName(sandboxName); + } catch { + return; + } + } + runApprovalPass(sandboxName, { + budget: CONNECT_AUTO_PAIR_BUDGET, + gatewayName: owningGatewayName, + }); } diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index df5cea7d022..376ab383747 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -59,7 +59,7 @@ describe("connectSandbox flow", () => { ); expect(harness.checkAndRecoverSpy).toHaveBeenCalledWith("alpha"); expect(harness.ensureOllamaAuthProxySpy).toHaveBeenCalledTimes(1); - expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha"); + expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha", "nemoclaw"); expect(harness.spawnSyncSpy).toHaveBeenCalledWith( "openshell", ["sandbox", "connect", "alpha"], @@ -71,6 +71,18 @@ describe("connectSandbox flow", () => { expect(exitSpy).toHaveBeenCalledWith(0); }); + it("uses the owning OpenShell gateway for auto-pair when an ambient gateway has the same sandbox name (#8942)", async () => { + vi.stubEnv("OPENSHELL_GATEWAY", "ambient-sibling"); + const harness = createConnectHarness({ + registryEntry: { gatewayName: "nemoclaw-8091", gatewayPort: 8091 }, + }); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(0)"); + + expect(process.env.OPENSHELL_GATEWAY).toBe("ambient-sibling"); + expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha", "nemoclaw-8091"); + }); + it("restores the terminal and prints reconnect guidance when SSH disconnects", async () => { const setRawModeSpy = vi.fn(); Object.defineProperty(process.stdin, "isTTY", { configurable: true, value: true }); @@ -521,6 +533,232 @@ describe("connectSandbox flow", () => { ); }); + it("probe-only accepts healthy launch evidence without duplicate recovery or publication (#8942)", async () => { + const sb = { name: "alpha", agent: "openclaw", provider: null, model: null, policies: [] }; + const harness = createConnectHarness({ + readinessDecision: { + kind: "accepted", + category: "accepted", + agent: { name: "openclaw" }, + sb, + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + expect(harness.ensureLiveSandboxSpy).not.toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.logSpy.mock.calls.flat().join("\n")).toContain( + "Probe complete: launch readiness is healthy for 'alpha'.", + ); + }); + + it("probe-only skips every mutation when a newer accepted lease replaces its epoch (#8942)", async () => { + const sb = { name: "alpha", agent: "openclaw", provider: null, model: null, policies: [] }; + const harness = createConnectHarness(); + harness.inspectLaunchReadinessSpy + .mockResolvedValueOnce({ + kind: "fallback", + category: "config", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }) + .mockResolvedValueOnce({ + kind: "accepted", + category: "accepted", + agent: { name: "openclaw" }, + sb, + }); + harness.launchReadinessMutationGateSpy.mockResolvedValueOnce({ kind: "changed" }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.inspectLaunchReadinessSpy).toHaveBeenCalledTimes(2); + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + expect(harness.ensureLiveSandboxSpy).not.toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.logSpy.mock.calls.flat().join("\n")).toContain( + "Probe complete: launch readiness is healthy for 'alpha'.", + ); + }); + + it("probe-only refuses runtime recovery when prior evidence cannot be fenced (#8942)", async () => { + const harness = createConnectHarness({ + readinessDecision: { + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: true, + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "complete probe and recovery did not run because prior launch-readiness evidence could not be fenced", + ); + }); + + it("probe-only reports failure to create new authority after secure absence is proven (#8942)", async () => { + const harness = createConnectHarness({ + readinessDecision: { + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: false, + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "no prior launch-readiness evidence can be accepted, but new launch-readiness authority could not be created", + ); + expect(harness.errorSpy.mock.calls.flat().join("\n")).not.toContain( + "prior launch-readiness evidence could not be fenced", + ); + }); + + it("probe-only completes macOS recovery before reporting unavailable evidence (#8942)", async () => { + const harness = createConnectHarness({ + readinessDecision: { + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: false, + authorityUnsupported: true, + }, + readinessPublicationResult: { kind: "evidence-failed" }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalledOnce(); + expect(harness.ensureLiveSandboxSpy).toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).toHaveBeenCalledOnce(); + expect(harness.errorSpy).toHaveBeenCalledWith( + " Probe failed: complete probe and recovery succeeded, but launch-readiness evidence is unavailable on this platform.", + ); + }); + + it("lets a public lifecycle command continue after recovery when evidence publication is unavailable (#8942)", async () => { + const harness = createConnectHarness({ + readinessDecision: { + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: false, + authorityUnsupported: true, + }, + readinessPublicationResult: { kind: "evidence-failed" }, + }); + + await expect( + harness.connectSandbox("alpha", { + probeOnly: true, + requireLaunchReadinessPublication: false, + }), + ).resolves.toBeUndefined(); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalledOnce(); + expect(harness.ensureLiveSandboxSpy).toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).toHaveBeenCalledOnce(); + expect(exitSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy).not.toHaveBeenCalled(); + }); + + it("keeps authoritative runtime validation failures blocking for public lifecycle commands (#8942)", async () => { + const harness = createConnectHarness({ + readinessPublicationResult: { kind: "validation-failed", category: "health" }, + }); + + await expect( + harness.connectSandbox("alpha", { + probeOnly: true, + requireLaunchReadinessPublication: false, + }), + ).rejects.toThrow("process.exit(1)"); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalledOnce(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "final launch-readiness validation failed due to health", + ); + }); + + it("probe-only stops before mutation when the fenced epoch cannot be revalidated (#8942)", async () => { + const harness = createConnectHarness(); + harness.launchReadinessMutationGateSpy.mockResolvedValueOnce({ kind: "unsafe" }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + expect(harness.ensureLiveSandboxSpy).not.toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "current launch-readiness epoch could not be safely revalidated", + ); + }); + + it("probe-only distinguishes completed recovery from final evidence failure (#8942)", async () => { + const harness = createConnectHarness({ + readinessPublicationResult: { kind: "evidence-failed" }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalled(); + expect(harness.publishLaunchReadinessSpy).toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "complete probe and recovery succeeded, but final launch-readiness evidence could not be verified or published", + ); + }); + + it("probe-only reports final semantic validation failure as a runtime failure (#8942)", async () => { + const harness = createConnectHarness({ + readinessPublicationResult: { kind: "validation-failed", category: "health" }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "final launch-readiness validation failed due to health", + ); + expect(harness.errorSpy.mock.calls.flat().join("\n")).not.toContain( + "complete probe and recovery succeeded", + ); + }); + it("probe-only mode exits before reporting success when inference.local returns no trusted result (#8502)", async () => { const harness = createConnectHarness({ registryEntry: { diff --git a/src/lib/actions/sandbox/connect-inference-route-probe.test.ts b/src/lib/actions/sandbox/connect-inference-route-probe.test.ts index 0e3e7e45703..4ea9fb74026 100644 --- a/src/lib/actions/sandbox/connect-inference-route-probe.test.ts +++ b/src/lib/actions/sandbox/connect-inference-route-probe.test.ts @@ -64,6 +64,15 @@ describe("sandbox connect inference route probe argv", () => { ]); }); + it.each([ + null, + { name: "langchain-deepagents-code" }, + ])("pins the probe to the owning OpenShell gateway for agent %j (#8942)", (agent) => { + expect( + buildSandboxInferenceRouteProbeArgs("alpha", agent, "nemoclaw-8091").slice(0, 7), + ).toEqual(["sandbox", "exec", "--name", "alpha", "-g", "nemoclaw-8091", expect.any(String)]); + }); + it("verifies the route with OpenShell's CA and discards the response (#6192)", () => { const args = buildSandboxInferenceRouteProbeArgs("alpha", { name: "openclaw" }); const script = args.at(-1) ?? ""; diff --git a/src/lib/actions/sandbox/connect-inference-route-probe.ts b/src/lib/actions/sandbox/connect-inference-route-probe.ts index 909c8806c3e..3c5730c3939 100644 --- a/src/lib/actions/sandbox/connect-inference-route-probe.ts +++ b/src/lib/actions/sandbox/connect-inference-route-probe.ts @@ -84,13 +84,18 @@ export function classifyInferenceRouteFailureLabel(httpStatus: number): Inferenc export function buildSandboxInferenceRouteProbeArgs( sandboxName: string, agent: InferenceRouteProbeAgent, + gatewayName?: string, ): string[] { + const targetArgs = [ + "sandbox", + "exec", + "--name", + sandboxName, + ...(gatewayName ? ["-g", gatewayName] : []), + ]; if (agent?.name === "langchain-deepagents-code") { return [ - "sandbox", - "exec", - "--name", - sandboxName, + ...targetArgs, "--no-tty", "--env", "HOME=/usr/local/lib/nemoclaw", @@ -109,7 +114,7 @@ export function buildSandboxInferenceRouteProbeArgs( ]; } - return ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-c", INFERENCE_ROUTE_PROBE_SCRIPT]; + return [...targetArgs, "--", "sh", "-c", INFERENCE_ROUTE_PROBE_SCRIPT]; } /** Parse the shared route-probe output used by connect, status, and doctor. */ diff --git a/src/lib/actions/sandbox/connect-route-lifecycle.test.ts b/src/lib/actions/sandbox/connect-route-lifecycle.test.ts index 74de681b538..65f1bb3a197 100644 --- a/src/lib/actions/sandbox/connect-route-lifecycle.test.ts +++ b/src/lib/actions/sandbox/connect-route-lifecycle.test.ts @@ -85,10 +85,10 @@ describe("connectSandbox route lifecycle", () => { }); it("repairs a WSL Ollama route without requiring an auth proxy token", async () => { - vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); const harness = createConnectHarness({ inferenceGetOutput: "Gateway inference:\n Provider: ollama-local\n Model: qwen3:0.6b\n", inferenceProbeResponses: ["BROKEN 503", "BROKEN 503", "OK 200", "OK 200"], + isWsl: true, registryEntry: { model: "qwen3:0.6b", provider: "ollama-local", diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 3d7cac3e955..aa780cc1e88 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -79,6 +79,12 @@ import { } from "./gateway-state"; import { getSandboxTargetGatewayName } from "./gateway-target"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; +import { + inspectLaunchReadiness, + publicationFromDecision, + publishLaunchReadiness, + withLaunchReadinessMutationGate, +} from "./launch-readiness"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand, @@ -94,6 +100,7 @@ export { runConnectAutoPairApprovalPass, waitForManagedGatewaySupervisor }; export type SandboxConnectOptions = { probeOnly?: boolean; + requireLaunchReadinessPublication?: boolean; }; export type SandboxStartupRecoveryResult = ReturnType & { @@ -1168,19 +1175,8 @@ async function runConnectEntryPreflight( } } -/** - * Everything an interactive sandbox session needs before SSH is spawned: - * the shared connect entry preflight plus process recovery, readiness wait, - * inference-route reconcile, and the auto-pair approval pass. Shared by - * `connect` and `launch`; both are always non-probe-only. Any - * `process.exit(...)` reached here ends the process exactly as it does on the - * connect path. - */ -export async function prepareInteractiveSession( - sandboxName: string, -): Promise<{ agent: AgentDefinition | null; sb: SandboxEntry | null }> { - await runConnectEntryPreflight(sandboxName, { probeOnly: false }); - +/** Print version and active-session hints on both interactive launch paths. */ +export function printInteractiveSessionHints(sandboxName: string): void { // Version staleness check — warn but don't block try { const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); @@ -1208,6 +1204,28 @@ export async function prepareInteractiveSession( } catch { /* non-fatal — don't block connect on session detection failure */ } +} + +/** Preserve session setup after the complete preflight or lease acceptance. */ +export function completeInteractiveSessionSetup( + sandboxName: string, + sb: SandboxEntry | null, +): void { + maybeEnsureHermesToolGatewayBroker(sb); + const gatewayName = sb ? resolveSandboxGatewayName(sb) : getSandboxTargetGatewayName(sandboxName); + runConnectAutoPairApprovalPass(sandboxName, gatewayName); +} + +/** + * Run the complete interactive preflight before SSH or agent launch, including + * process recovery, readiness polling, inference-route repair, and session + * setup. Any `process.exit(...)` ends the process as it does on `connect`. + */ +export async function prepareInteractiveSession( + sandboxName: string, +): Promise<{ agent: AgentDefinition | null; sb: SandboxEntry | null }> { + await runConnectEntryPreflight(sandboxName, { probeOnly: false }); + printInteractiveSessionHints(sandboxName); const processCheck = checkAndRecoverSandboxProcesses(sandboxName); if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { @@ -1233,36 +1251,75 @@ export async function prepareInteractiveSession( // After the sandbox is Ready, verify and recover the route before SSH. const agent = agentRuntime.getSessionAgent(sandboxName); sb = await ensureSandboxInferenceRouteOrExit(sandboxName, agent); - maybeEnsureHermesToolGatewayBroker(sb); - - // ── Auto-pair late scope-upgrade approval (#4263) ─────────────── - // Defense in depth: even with the in-sandbox watcher running in - // slow-mode keepalive, a brief approval pass before opening SSH - // catches any pending allowlisted CLI/webchat scope upgrades that - // piled up between startup and now (e.g., watcher crashed, watcher - // deadline exhausted, multi-sandbox gateway contention). The same pass - // is reachable without SSH via `doctor --fix` for dashboard-only users - // (#4616). Uses the tight connect budget (#4504). - runConnectAutoPairApprovalPass(sandboxName); + completeInteractiveSessionSetup(sandboxName, sb); return { agent, sb }; } export async function connectSandbox( sandboxName: string, - { probeOnly = false }: SandboxConnectOptions = {}, + { probeOnly = false, requireLaunchReadinessPublication = true }: SandboxConnectOptions = {}, ): Promise { if (probeOnly) { - await runConnectEntryPreflight(sandboxName, { probeOnly: true }); - waitForSandboxReadyOrExit(sandboxName, { - defaultTimeoutSec: 300, - retryCommand: "connect --probe-only", - }); - // Re-pin and re-observe the owning gateway after a potentially long wait - // before any in-sandbox process or host-forward mutation. The readiness - // polls are already owner-scoped; this also catches registry changes. - await ensureLiveSandboxOrExit(sandboxName, { gatewayRecovery: "observe" }); - return await runSandboxConnectProbe(sandboxName); + let readiness = await inspectLaunchReadiness(sandboxName); + let publication: Awaited>; + while (true) { + if (readiness.kind === "accepted") { + console.log(` Probe complete: launch readiness is healthy for '${sandboxName}'.`); + return; + } + if (readiness.fenceFailed && readiness.authorityUnsupported !== true) { + console.error( + readiness.recoveryBlocked + ? " Probe failed: complete probe and recovery did not run because prior launch-readiness evidence could not be fenced. Repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry." + : " Probe failed: no prior launch-readiness evidence can be accepted, but new launch-readiness authority could not be created. Repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry.", + ); + process.exit(1); + } + const publicationRequest = publicationFromDecision(sandboxName, readiness); + const gated = await withLaunchReadinessMutationGate(publicationRequest, async () => { + await runConnectEntryPreflight(sandboxName, { probeOnly: true }); + waitForSandboxReadyOrExit(sandboxName, { + defaultTimeoutSec: 300, + retryCommand: "connect --probe-only", + }); + // Re-pin and re-observe the owning gateway after a potentially long wait + // before any in-sandbox process or host-forward mutation. The readiness + // polls are already scoped to the owning gateway; this also catches + // registry changes. + await ensureLiveSandboxOrExit(sandboxName, { gatewayRecovery: "observe" }); + await runSandboxConnectProbe(sandboxName); + return publishLaunchReadiness(publicationRequest); + }); + if (gated.kind === "changed") { + readiness = await inspectLaunchReadiness(sandboxName); + continue; + } + if (gated.kind === "unsafe") { + console.error( + " Probe failed: complete probe and recovery did not run because the current launch-readiness epoch could not be safely revalidated. Repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry.", + ); + process.exit(1); + } + publication = gated.value; + break; + } + if (publication.kind === "validation-failed") { + console.error( + ` Probe failed: final launch-readiness validation failed due to ${publication.category}.`, + ); + process.exit(1); + } + if (publication.kind === "evidence-failed") { + if (!requireLaunchReadinessPublication) return; + console.error( + readiness.kind === "fallback" && readiness.authorityUnsupported === true + ? " Probe failed: complete probe and recovery succeeded, but launch-readiness evidence is unavailable on this platform." + : " Probe failed: complete probe and recovery succeeded, but final launch-readiness evidence could not be verified or published.", + ); + process.exit(1); + } + return; } const { agent, sb } = await prepareInteractiveSession(sandboxName); diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index f16e7754562..d93d2fe59fa 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -493,6 +493,58 @@ export function ensureDeclaredAgentForwardPortsHealthy( return allHealthy; } +/** + * Observe every host forward that the interactive preflight would recover, + * without starting, stopping, or rebinding one. + */ +export function areSandboxLaunchForwardsHealthy( + sandboxName: string, + gatewayName?: string, +): boolean | null { + const sandbox = registry.getSandbox(sandboxName); + if (!sandbox) return false; + const owningGatewayName = resolveSandboxGatewayName(sandbox); + if (gatewayName && gatewayName !== owningGatewayName) return false; + const agent = agentRuntime.getSessionAgent(sandboxName); + if (agent && !agentRuntime.hasGatewayRuntime(agent)) return true; + + const primaryPort = resolveSandboxDashboardPort(sandboxName); + const requiredPorts = new Set([primaryPort]); + const hermesDashboard = getHermesDashboardRecoveryConfig(sandboxName); + if (hermesDashboard) requiredPorts.add(hermesDashboard.publicPort); + const messagingForward = getSandboxMessagingHostForward(sandboxName); + if (messagingForward) requiredPorts.add(messagingForward.port); + const declared = (agent as { forward_ports?: unknown } | null)?.forward_ports; + if (Array.isArray(declared)) { + for (const candidate of declared) { + if ( + typeof candidate === "number" && + Number.isInteger(candidate) && + candidate >= 1024 && + candidate <= 65535 + ) { + requiredPorts.add(candidate); + } + } + } + const result = captureOpenshell(["forward", "list", "--gateway", owningGatewayName], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + if (!result || isCommandTimeout(result) || result.status !== 0) return null; + const entries = parseForwardList(result.output) as SandboxForwardListEntry[]; + for (const port of requiredPorts) { + if ( + classifyForwardHealthWithReachability(entries, sandboxName, String(port), () => + isLocalForwardReachable(port), + ) !== true + ) { + return false; + } + } + return true; +} + export function recoverDeclaredAgentForwardPorts( sandboxName: string, recoveryPort: number, diff --git a/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts b/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts new file mode 100644 index 00000000000..0d922892b3c --- /dev/null +++ b/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { isSandboxGatewayRunningForStatus } from "./process-recovery"; + +describe("launch-readiness gateway health scope", () => { + it("pins the semantic gateway probe to the owning OpenShell gateway (#8942)", async () => { + const capture = vi.fn(async (_args: string[]) => ({ + status: 0, + output: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + })); + + await expect( + isSandboxGatewayRunningForStatus("alpha", "nemoclaw-8091", { + getSessionAgent: () => null, + getHealthProbeUrl: () => "http://127.0.0.1:18789/health", + capture: capture as never, + }), + ).resolves.toBe(true); + + expect(capture).toHaveBeenCalledTimes(1); + expect(capture.mock.calls[0]?.[0]?.slice(0, 7)).toEqual([ + "sandbox", + "exec", + "--name", + "alpha", + "-g", + "nemoclaw-8091", + "--", + ]); + }); +}); diff --git a/src/lib/actions/sandbox/launch-readiness.test.ts b/src/lib/actions/sandbox/launch-readiness.test.ts new file mode 100644 index 00000000000..164fb5821e3 --- /dev/null +++ b/src/lib/actions/sandbox/launch-readiness.test.ts @@ -0,0 +1,1176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { loadAgent } from "../../agent/defs"; +import type { + LaunchReadinessFence, + LaunchReadinessIdentity, + LaunchReadinessLease, +} from "../../state/launch-readiness-lease"; +import { LaunchReadinessFenceError } from "../../state/launch-readiness-lease"; +import type { SandboxEntry } from "../../state/registry"; +import { + buildLaunchReadinessRegistryProjection, + inspectLaunchReadiness, + type LaunchReadinessDeps, + launchReadinessDigest, + launchReadinessPolicyDigest, + publicationFromDecision, + publishLaunchReadiness, + withLaunchReadinessMutationGate, +} from "./launch-readiness"; + +const SANDBOX = "alpha"; +const GATEWAY_NAME = "nemoclaw"; +const GATEWAY_PORT = 8080; +const EPOCH = "a".repeat(64); +const FINGERPRINT = "b".repeat(64); +const DIGEST = "c".repeat(64); + +const POLICY_A = `version: 1 +network_policies: + public_api: + name: Public API + endpoints: + - host: example.com + port: 443 + binaries: + - path: /usr/bin/curl +`; + +const POLICY_A_REORDERED = `network_policies: + public_api: + binaries: + - path: /usr/bin/curl + endpoints: + - port: 443 + host: example.com + name: Public API +version: 1 +`; + +const POLICY_B = POLICY_A.replace("example.com", "api.example.com"); + +function entry(agent = "openclaw"): SandboxEntry { + return { + name: SANDBOX, + openshellDriver: "docker", + openshellVersion: "0.0.99", + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: FINGERPRINT, + agent, + agentVersion: "1.0.0", + nemoclawVersion: "2.0.0", + imageTag: "example@sha256:immutable", + policyPresetsFinalized: true, + policies: ["managed_inference"], + policyTier: "standard", + provider: null, + model: null, + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + }; +} + +function servingProfile(): NonNullable { + return { + schemaVersion: 1, + catalogDigest: `sha256:${"d".repeat(64)}`, + preset: { + id: "local-gpu", + digest: `sha256:${"e".repeat(64)}`, + displayName: "Local GPU", + supportState: "supported", + }, + recipe: { + id: "vllm-local", + digest: `sha256:${"f".repeat(64)}`, + backend: "vllm", + }, + model: { id: "model-a", revision: "revision-a" }, + runtimeImage: "example.com/runtime@sha256:immutable", + estimatedImageDownloadBytes: 1_000, + estimatedModelDownloadBytes: 2_000, + }; +} + +function fence(): LaunchReadinessFence { + return { + schemaVersion: 1, + kind: "fence", + epochId: EPOCH, + sandboxName: SANDBOX, + fencedWallMs: 1, + fencedUptimeMs: 1, + bootId: "boot-a", + uid: 1, + homeDevice: "1", + homeInode: "2", + storeDevice: "1", + storeInode: "3", + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + publicationState: "ready", + preservedLeaseStartedWallMs: null, + preservedLeaseExpiresWallMs: null, + preservedLeaseElapsedMs: null, + }; +} + +function lease(identity: LaunchReadinessIdentity): LaunchReadinessLease { + return { + schemaVersion: 1, + kind: "lease", + epochId: EPOCH, + sandboxName: SANDBOX, + leaseStartedWallMs: 1, + leaseExpiresWallMs: 86_400_001, + elapsedAtPublicationMs: 0, + publishedWallMs: 1, + publishedUptimeMs: 1, + bootId: "boot-a", + uid: 1, + homeDevice: "1", + homeInode: "2", + storeDevice: "1", + storeInode: "3", + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + identity, + }; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +function serialTestLock( + events: string[], + label: string, +): NonNullable { + let tail = Promise.resolve(); + return async (_name: string, operation: () => Promise | T): Promise => { + const previous = tail; + const release = deferred(); + tail = previous.then(() => release.promise); + await previous; + events.push(`${label}:start`); + try { + return await operation(); + } finally { + events.push(`${label}:end`); + release.resolve(); + } + }; +} + +describe("launch readiness validation", () => { + let sandbox: SandboxEntry; + let policy: string; + let routeOutput: string; + let readKind: "missing" | "valid"; + let publishedIdentity: LaunchReadinessIdentity | null; + let runtimeHealthy: boolean | null; + let forwardsHealthy: boolean | null; + let observedFingerprint: string; + let lockEvents: string[]; + let externalEvents: string[]; + let observationRequests: Array<{ + sandboxName: string; + gatewayName: string; + gatewayPort: number; + }>; + let captureRequests: string[][]; + let gatewayHealthRequests: Array<[string, string]>; + let forwardRequests: Array<[string, string]>; + let inferenceHealthRequests: Array<[string, string]>; + + beforeEach(() => { + sandbox = entry(); + policy = POLICY_A; + routeOutput = "Gateway Inference:\n\n Not configured\n"; + readKind = "missing"; + publishedIdentity = null; + runtimeHealthy = true; + forwardsHealthy = true; + observedFingerprint = FINGERPRINT; + lockEvents = []; + externalEvents = []; + observationRequests = []; + captureRequests = []; + gatewayHealthRequests = []; + forwardRequests = []; + inferenceHealthRequests = []; + performance.clearMeasures("nemoclaw.launch-readiness.storage-read"); + performance.clearMeasures("nemoclaw.launch-readiness.live-validation"); + performance.clearMeasures("nemoclaw.launch-readiness.evidence-fence"); + performance.clearMeasures("nemoclaw.launch-readiness.publication-validation"); + performance.clearMeasures("nemoclaw.launch-readiness.publication-store"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + function deps(): LaunchReadinessDeps { + return { + getSandbox: () => sandbox, + listAgents: () => ["openclaw", "langchain-deepagents-code"], + loadAgent, + observeSandbox: (request) => { + externalEvents.push("sandbox-get"); + observationRequests.push(request); + return { + state: "ready", + liveIdentityFingerprint: observedFingerprint, + }; + }, + capture: (args) => { + externalEvents.push(args[0] === "policy" ? "policy-get" : "inference-get"); + captureRequests.push([...args]); + return { + status: 0, + output: args[0] === "policy" ? policy : routeOutput, + stdout: args[0] === "policy" ? policy : routeOutput, + stderr: "", + } as ReturnType>; + }, + gatewayHealth: async (sandboxName, gatewayName) => { + externalEvents.push("gateway-health"); + gatewayHealthRequests.push([sandboxName, gatewayName]); + return runtimeHealthy; + }, + forwardsHealthy: (sandboxName, gatewayName) => { + externalEvents.push("forward-list"); + forwardRequests.push([sandboxName, gatewayName]); + return forwardsHealthy; + }, + smoke: () => ({ ok: true }), + inferenceProbe: (sandboxName, _agent, gatewayName) => { + externalEvents.push("inference-health"); + inferenceHealthRequests.push([sandboxName, gatewayName]); + return { healthy: true, broken: false, httpStatus: 200, detail: "OK 200" }; + }, + readLease: () => + readKind === "valid" && publishedIdentity + ? { kind: "valid", lease: lease(publishedIdentity) } + : { kind: "missing" }, + fenceLease: () => fence(), + publishLease: (_name, _gateway, _port, _epoch, identity) => { + publishedIdentity = identity; + return lease(identity); + }, + withSandboxLock: async (_name: string, operation: () => Promise | T) => { + lockEvents.push("sandbox:start"); + const result = await operation(); + lockEvents.push("sandbox:end"); + return result; + }, + withGatewayLock: async (_name: string, operation: () => Promise | T) => { + lockEvents.push("gateway:start"); + const result = await operation(); + lockEvents.push("gateway:end"); + return result; + }, + }; + } + + async function createAcceptedLease(currentDeps = deps()) { + const first = await inspectLaunchReadiness(SANDBOX, currentDeps); + expect(first).toMatchObject({ kind: "fallback", category: "missing", fenceFailed: false }); + expect( + await publishLaunchReadiness(publicationFromDecision(SANDBOX, first), currentDeps), + ).toEqual({ kind: "published" }); + expect(publishedIdentity).not.toBeNull(); + readKind = "valid"; + lockEvents = []; + return currentDeps; + } + + it("accepts only after final capture and follows sandbox then gateway lock order", async () => { + const currentDeps = await createAcceptedLease(); + externalEvents = []; + const decision = await inspectLaunchReadiness(SANDBOX, currentDeps); + expect(decision).toMatchObject({ kind: "accepted", category: "accepted" }); + expect(lockEvents).toEqual(["sandbox:start", "gateway:start", "gateway:end", "sandbox:end"]); + expect(externalEvents).toEqual([ + "sandbox-get", + "policy-get", + "inference-get", + "gateway-health", + "forward-list", + ]); + }); + + it("uses a fenced CAS epoch when secure authority is available (#8942)", async () => { + const decision = await inspectLaunchReadiness(SANDBOX, deps()); + + expect(decision).toMatchObject({ + kind: "fallback", + fence: { epochId: EPOCH }, + fenceFailed: false, + recoveryBlocked: false, + }); + expect(decision).not.toHaveProperty("authorityUnsupported"); + expect(publicationFromDecision(SANDBOX, decision)).toMatchObject({ epochId: EPOCH }); + }); + + it("revalidates the producer epoch under both canonical locks before mutation", async () => { + const currentDeps = deps(); + const checkMutationAuthority = vi.fn(() => "current" as const); + currentDeps.checkMutationAuthority = checkMutationAuthority; + lockEvents = []; + + const result = await withLaunchReadinessMutationGate( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: EPOCH, + }, + () => { + lockEvents.push("mutation"); + return "complete"; + }, + currentDeps, + ); + + expect(result).toEqual({ kind: "entered", value: "complete" }); + expect(checkMutationAuthority).toHaveBeenCalledWith( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + EPOCH, + undefined, + ); + expect(lockEvents).toEqual([ + "sandbox:start", + "gateway:start", + "mutation", + "gateway:end", + "sandbox:end", + ]); + }); + + it("rejects a stale fenced epoch before entering the mutation callback (#8942)", async () => { + const currentDeps = deps(); + const checkMutationAuthority = vi.fn(() => "changed" as const); + const mutation = vi.fn(); + currentDeps.checkMutationAuthority = checkMutationAuthority; + + await expect( + withLaunchReadinessMutationGate( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: EPOCH, + }, + mutation, + currentDeps, + ), + ).resolves.toEqual({ kind: "changed" }); + expect(checkMutationAuthority).toHaveBeenCalledWith( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + EPOCH, + undefined, + ); + expect(mutation).not.toHaveBeenCalled(); + }); + + it("delays a later producer's epoch rotation until the current producer releases the mutation locks", async () => { + const events: string[] = []; + const sandboxLock = serialTestLock(events, "sandbox"); + const gatewayLock = serialTestLock(events, "gateway"); + const mutationEntered = deferred(); + const releaseMutation = deferred(); + const fenceB = vi.fn(() => { + events.push("producer-b:rotate"); + return { ...fence(), epochId: "b".repeat(64) }; + }); + const producerA = withLaunchReadinessMutationGate( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: EPOCH, + }, + async () => { + events.push("producer-a:mutation-start"); + mutationEntered.resolve(); + await releaseMutation.promise; + events.push("producer-a:mutation-end"); + }, + { + ...deps(), + checkMutationAuthority: () => "current", + withSandboxLock: sandboxLock, + withGatewayLock: gatewayLock, + }, + ); + await mutationEntered.promise; + + const producerB = inspectLaunchReadiness(SANDBOX, { + ...deps(), + readLease: () => ({ kind: "missing" }), + fenceLease: fenceB, + withSandboxLock: sandboxLock, + withGatewayLock: gatewayLock, + }); + await Promise.resolve(); + expect(fenceB).not.toHaveBeenCalled(); + + releaseMutation.resolve(); + await expect(producerA).resolves.toMatchObject({ kind: "entered" }); + await expect(producerB).resolves.toMatchObject({ + kind: "fallback", + fence: { epochId: "b".repeat(64) }, + }); + expect(events.indexOf("producer-a:mutation-end")).toBeLessThan( + events.indexOf("producer-b:rotate"), + ); + }); + + it("distinguishes blocking authority failures from evidence-free storage failure", async () => { + const blockedDeps = deps(); + blockedDeps.fenceLease = () => { + throw new LaunchReadinessFenceError(true, false); + }; + await expect(inspectLaunchReadiness(SANDBOX, blockedDeps)).resolves.toMatchObject({ + kind: "fallback", + fenceFailed: true, + recoveryBlocked: true, + }); + + const unavailableDeps = deps(); + unavailableDeps.fenceLease = () => { + throw new LaunchReadinessFenceError(false, false); + }; + await expect(inspectLaunchReadiness(SANDBOX, unavailableDeps)).resolves.toMatchObject({ + kind: "fallback", + fenceFailed: true, + recoveryBlocked: false, + }); + }); + + it("records only accepted-path stages without wall-clock pass thresholds", async () => { + const currentDeps = await createAcceptedLease(); + performance.clearMeasures("nemoclaw.launch-readiness.storage-read"); + performance.clearMeasures("nemoclaw.launch-readiness.live-validation"); + performance.clearMeasures("nemoclaw.launch-readiness.evidence-fence"); + performance.clearMeasures("nemoclaw.launch-readiness.publication-validation"); + performance.clearMeasures("nemoclaw.launch-readiness.publication-store"); + await inspectLaunchReadiness(SANDBOX, currentDeps); + + const names = performance + .getEntriesByType("measure") + .map((entry) => entry.name) + .filter((name) => name.startsWith("nemoclaw.launch-readiness.")); + expect(new Set(names)).toEqual( + new Set([ + "nemoclaw.launch-readiness.storage-read", + "nemoclaw.launch-readiness.live-validation", + ]), + ); + }); + + it("fences config, policy, live identity, and health changes before fallback", async () => { + const currentDeps = await createAcceptedLease(); + const cases: Array<{ + category: "config" | "identity" | "health"; + mutate: () => void; + restore: () => void; + }> = [ + { + category: "config", + mutate: () => { + sandbox = { ...sandbox, policyTier: "strict" }; + }, + restore: () => { + sandbox = { ...sandbox, policyTier: "standard" }; + }, + }, + { + category: "config", + mutate: () => { + policy = POLICY_B; + }, + restore: () => { + policy = POLICY_A; + }, + }, + { + category: "identity", + mutate: () => { + observedFingerprint = DIGEST; + }, + restore: () => { + observedFingerprint = FINGERPRINT; + }, + }, + { + category: "health", + mutate: () => { + runtimeHealthy = false; + }, + restore: () => { + runtimeHealthy = true; + }, + }, + { + category: "health", + mutate: () => { + forwardsHealthy = false; + }, + restore: () => { + forwardsHealthy = true; + }, + }, + ]; + for (const testCase of cases) { + testCase.mutate(); + const decision = await inspectLaunchReadiness(SANDBOX, currentDeps); + expect(decision).toMatchObject({ + kind: "fallback", + category: testCase.category, + fence: { epochId: EPOCH }, + fenceFailed: false, + }); + testCase.restore(); + } + }); + + it("requires exact owning-gateway policy, inference route, and semantic health", async () => { + vi.stubEnv("OPENSHELL_GATEWAY", "ambient-sibling"); + sandbox = { + ...sandbox, + provider: "nvidia", + model: "model-a", + credentialEnv: "NVIDIA_API_KEY", + }; + routeOutput = "Gateway Inference:\n\n Provider: nvidia\n Model: model-a\n"; + const currentDeps = await createAcceptedLease(); + externalEvents = []; + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ kind: "accepted" }); + expect(externalEvents).toEqual([ + "sandbox-get", + "policy-get", + "inference-get", + "gateway-health", + "forward-list", + "inference-health", + ]); + expect(observationRequests).toContainEqual({ + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + }); + expect(captureRequests).toContainEqual([ + "policy", + "get", + "-g", + GATEWAY_NAME, + "--full", + SANDBOX, + ]); + expect(captureRequests).toContainEqual(["inference", "get", "-g", GATEWAY_NAME]); + expect(gatewayHealthRequests).toContainEqual([SANDBOX, GATEWAY_NAME]); + expect(forwardRequests).toContainEqual([SANDBOX, GATEWAY_NAME]); + expect(inferenceHealthRequests).toContainEqual([SANDBOX, GATEWAY_NAME]); + expect(process.env.OPENSHELL_GATEWAY).toBe("ambient-sibling"); + routeOutput = "Gateway Inference:\n\n Provider: nvidia\n Model: model-b\n"; + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ + kind: "fallback", + category: "config", + }); + + routeOutput = "Gateway Inference:\n\n Provider: nvidia\n Model: model-a\n"; + currentDeps.inferenceProbe = () => ({ + healthy: false, + broken: true, + httpStatus: 503, + detail: "BROKEN 503", + }); + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ + kind: "fallback", + category: "health", + }); + }); + + it.each([300, 401, 403, 404, 503])( + "rejects HTTP %i from the owning OpenShell gateway inference probe during inspection and publication (#8942)", + async (httpStatus) => { + sandbox = { + ...sandbox, + provider: "nvidia", + model: "model-a", + credentialEnv: "NVIDIA_API_KEY", + }; + routeOutput = "Gateway Inference:\n\n Provider: nvidia\n Model: model-a\n"; + const currentDeps = await createAcceptedLease(); + currentDeps.inferenceProbe = vi.fn((_sandboxName, _agent, _gatewayName) => ({ + healthy: httpStatus < 500, + broken: httpStatus >= 500, + httpStatus, + detail: `${httpStatus < 500 ? "OK" : "BROKEN"} ${httpStatus}`, + })); + + await expect(inspectLaunchReadiness(SANDBOX, currentDeps)).resolves.toMatchObject({ + kind: "fallback", + category: "health", + }); + await expect( + publishLaunchReadiness( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: EPOCH, + }, + currentDeps, + ), + ).resolves.toEqual({ kind: "validation-failed", category: "health" }); + expect(currentDeps.inferenceProbe).toHaveBeenCalledWith( + SANDBOX, + expect.objectContaining({ name: "openclaw" }), + GATEWAY_NAME, + ); + }, + ); + + it("accepts strict HTTP 2xx inference evidence from the owning OpenShell gateway (#8942)", async () => { + sandbox = { + ...sandbox, + provider: "nvidia", + model: "model-a", + credentialEnv: "NVIDIA_API_KEY", + }; + routeOutput = "Gateway Inference:\n\n Provider: nvidia\n Model: model-a\n"; + const currentDeps = await createAcceptedLease(); + currentDeps.inferenceProbe = vi.fn((_sandboxName, _agent, gatewayName) => ({ + healthy: true, + broken: false, + httpStatus: 299, + detail: "OK 299", + })); + + await expect(inspectLaunchReadiness(SANDBOX, currentDeps)).resolves.toMatchObject({ + kind: "accepted", + category: "accepted", + }); + await expect( + publishLaunchReadiness( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: EPOCH, + }, + currentDeps, + ), + ).resolves.toEqual({ kind: "published" }); + expect(currentDeps.inferenceProbe).toHaveBeenCalledWith( + SANDBOX, + expect.objectContaining({ name: "openclaw" }), + GATEWAY_NAME, + ); + }); + + it("rejects a caller-controlled OpenShell gateway endpoint", async () => { + const currentDeps = await createAcceptedLease(); + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://attacker.invalid"); + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ + kind: "fallback", + category: "config", + fence: { epochId: EPOCH }, + }); + }); + + it("uses terminal-agent smoke health for a supported non-OpenClaw runtime", async () => { + sandbox = entry("langchain-deepagents-code"); + const currentDeps = deps(); + const gatewayHealth = vi.fn(async () => true); + const smoke = vi.fn(() => ({ ok: true }) as const); + currentDeps.gatewayHealth = gatewayHealth; + currentDeps.smoke = smoke; + await createAcceptedLease(currentDeps); + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ kind: "accepted" }); + expect(smoke).toHaveBeenCalledWith( + SANDBOX, + expect.objectContaining({ name: "langchain-deepagents-code" }), + expect.any(Function), + GATEWAY_NAME, + ); + expect(gatewayHealth).not.toHaveBeenCalled(); + }); + + it("uses the normalized trusted agent name for CUA semantic health (#8942)", async () => { + sandbox = entry(" nemocua "); + const currentDeps = deps(); + const cuaReadiness = vi.fn(); + const gatewayHealth = vi.fn(async () => true); + const smoke = vi.fn(() => ({ ok: true }) as const); + const cuaAgent = { + ...loadAgent("hermes"), + name: "nemocua", + runtime: { + kind: "terminal" as const, + interactive_command: "nemocua interactive", + headless_command: "nemocua headless", + }, + }; + currentDeps.listAgents = () => ["nemocua"]; + currentDeps.loadAgent = () => cuaAgent; + currentDeps.cuaReadiness = cuaReadiness; + currentDeps.gatewayHealth = gatewayHealth; + currentDeps.smoke = smoke; + + await createAcceptedLease(currentDeps); + expect(await inspectLaunchReadiness(SANDBOX, currentDeps)).toMatchObject({ kind: "accepted" }); + expect(cuaReadiness).toHaveBeenCalledTimes(2); + expect(smoke).not.toHaveBeenCalled(); + expect(gatewayHealth).not.toHaveBeenCalled(); + }); + + it("hashes parsed policy semantics instead of presentation bytes", () => { + expect(launchReadinessPolicyDigest(POLICY_A_REORDERED)).toBe( + launchReadinessPolicyDigest(POLICY_A), + ); + expect(launchReadinessPolicyDigest(POLICY_B)).not.toBe(launchReadinessPolicyDigest(POLICY_A)); + }); + + it("uses an exact versioned allowlist for launch-affecting registry state", () => { + const agent = loadAgent("openclaw"); + const projection = buildLaunchReadinessRegistryProjection(sandbox, agent) as Record< + string, + unknown + >; + expect(Object.keys(projection).sort()).toEqual( + [ + "agent", + "agentVersion", + "baselineExclusions", + "cuaRuntimeReadinessSha256", + "customPolicies", + "dashboardPort", + "dashboardRemoteBindPrepared", + "dcodeAutoApprovalMode", + "fromDockerfile", + "gatewayName", + "gatewayPort", + "gpuEnabled", + "hermesAuthMethod", + "hermesDashboardEnabled", + "hermesDashboardInternalPort", + "hermesDashboardPort", + "hermesDashboardTui", + "hermesInferenceProvider", + "hermesToolGateways", + "hostGpuDetected", + "hostMounts", + "imageTag", + "inference", + "interactiveCommand", + "lifecycleGeneration", + "lifecycleLiveIdentityFingerprint", + "mcpSha256", + "messagingSha256", + "name", + "nemoclawVersion", + "observabilityEnabled", + "openclawImagePluginInstalls", + "openshellDriver", + "openshellVersion", + "policies", + "policyPresetsFinalized", + "policyTier", + "sandboxGpuDevice", + "sandboxGpuEnabled", + "sandboxGpuMode", + "sandboxGpuProof", + "servingProfileProvenance", + "toolDisclosure", + "version", + "webSearchEnabled", + "webSearchProvider", + "workloadIdentitySha256", + ].sort(), + ); + expect(projection.version).toBe(2); + const original = launchReadinessDigest(projection); + const mutations: SandboxEntry[] = [ + { ...sandbox, nemoclawVersion: "changed" }, + { + ...sandbox, + hostMounts: [ + { + source: "/private/host/project", + target: "/sandbox/project", + readOnly: true, + sourceIdentity: { device: "1", inode: "2" }, + }, + ], + }, + { ...sandbox, gpuEnabled: true }, + { ...sandbox, hostGpuDetected: true }, + { ...sandbox, sandboxGpuEnabled: true }, + { ...sandbox, sandboxGpuMode: "1" }, + { ...sandbox, sandboxGpuDevice: "0" }, + { ...sandbox, servingProfileProvenance: servingProfile() }, + { ...sandbox, hermesAuthMethod: "oauth" }, + { ...sandbox, policies: ["managed_inference", "slack"] }, + { + ...sandbox, + customPolicies: [{ name: "custom", content: POLICY_A, pendingContent: POLICY_B }], + }, + { + ...sandbox, + baselineExclusions: [ + { + version: 1, + agent: "openclaw", + key: "phone_home", + digest: DIGEST, + appliedAgentVersion: "1.0.0", + }, + ], + }, + { ...sandbox, webSearchEnabled: true, webSearchProvider: "brave" }, + { ...sandbox, observabilityEnabled: true }, + { ...sandbox, hermesDashboardEnabled: true, hermesDashboardPort: 3000 }, + { ...sandbox, dashboardRemoteBindPrepared: true }, + { + ...sandbox, + sandboxGpuProof: { + status: "verified", + cudaVerified: true, + label: "cuda", + at: "2026-01-01T00:00:00.000Z", + }, + }, + { + ...sandbox, + openclawImagePluginInstalls: [ + { id: "plugin", installPath: "/sandbox/.openclaw/extensions/plugin", loadPaths: [] }, + ], + }, + ]; + for (const mutation of mutations) { + expect( + launchReadinessDigest(buildLaunchReadinessRegistryProjection(mutation, agent)), + ).not.toBe(original); + } + }); + + it("binds every host mount field without projecting the host source path (#8942)", () => { + const agent = loadAgent("openclaw"); + const source = "/private/host/customer-project"; + const mounted: SandboxEntry = { + ...sandbox, + hostMounts: [ + { + source, + target: "/sandbox/project", + readOnly: true, + sourceIdentity: { device: "11", inode: "22" }, + }, + ], + }; + const projection = buildLaunchReadinessRegistryProjection(mounted, agent) as { + hostMounts: Array>; + }; + expect(JSON.stringify(projection)).not.toContain(source); + expect(projection.hostMounts).toEqual([ + { + sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + target: "/sandbox/project", + readOnly: true, + sourceIdentity: { device: "11", inode: "22" }, + }, + ]); + + const original = launchReadinessDigest(projection); + const mutations: SandboxEntry[] = [ + { + ...mounted, + hostMounts: [{ ...mounted.hostMounts![0]!, source: `${source}-changed` }], + }, + { + ...mounted, + hostMounts: [{ ...mounted.hostMounts![0]!, target: "/sandbox/changed" }], + }, + { + ...mounted, + hostMounts: [ + { + ...mounted.hostMounts![0]!, + sourceIdentity: { device: "12", inode: "22" }, + }, + ], + }, + { + ...mounted, + hostMounts: [ + { + ...mounted.hostMounts![0]!, + sourceIdentity: { device: "11", inode: "23" }, + }, + ], + }, + ]; + for (const mutation of mutations) { + expect( + launchReadinessDigest(buildLaunchReadinessRegistryProjection(mutation, agent)), + ).not.toBe(original); + } + expect(() => + buildLaunchReadinessRegistryProjection( + { + ...mounted, + hostMounts: [{ ...mounted.hostMounts![0]!, readOnly: false as true }], + }, + agent, + ), + ).toThrow(); + }); + + it("binds every semantic serving profile provenance field (#8942)", () => { + const agent = loadAgent("openclaw"); + const originalProfile = servingProfile(); + const original = launchReadinessDigest( + buildLaunchReadinessRegistryProjection( + { ...sandbox, servingProfileProvenance: originalProfile }, + agent, + ), + ); + const mutations: NonNullable[] = [ + { ...originalProfile, catalogDigest: `sha256:${"a".repeat(64)}` }, + { ...originalProfile, preset: { ...originalProfile.preset, id: "changed" } }, + { + ...originalProfile, + preset: { ...originalProfile.preset, digest: `sha256:${"a".repeat(64)}` }, + }, + { ...originalProfile, preset: { ...originalProfile.preset, displayName: "Changed" } }, + { + ...originalProfile, + preset: { ...originalProfile.preset, supportState: "experimental" }, + }, + { ...originalProfile, recipe: { ...originalProfile.recipe, id: "changed" } }, + { + ...originalProfile, + recipe: { ...originalProfile.recipe, digest: `sha256:${"a".repeat(64)}` }, + }, + { ...originalProfile, recipe: { ...originalProfile.recipe, backend: "changed" } }, + { ...originalProfile, model: { ...originalProfile.model, id: "changed" } }, + { ...originalProfile, model: { ...originalProfile.model, revision: "changed" } }, + { ...originalProfile, runtimeImage: "example.com/changed@sha256:immutable" }, + { ...originalProfile, estimatedImageDownloadBytes: 1_001 }, + { ...originalProfile, estimatedModelDownloadBytes: 2_001 }, + ]; + for (const mutation of mutations) { + expect( + launchReadinessDigest( + buildLaunchReadinessRegistryProjection( + { ...sandbox, servingProfileProvenance: mutation }, + agent, + ), + ), + ).not.toBe(original); + } + }); + + it("excludes diagnostic timestamps, source paths, and GPU detail from the projection", () => { + const agent = loadAgent("openclaw"); + const first: SandboxEntry = { + ...sandbox, + createdAt: "2026-01-01T00:00:00.000Z", + customPolicies: [ + { + name: "custom", + content: POLICY_A, + sourcePath: "/first/policy.yaml", + appliedAt: "2026-01-01T00:00:00.000Z", + }, + ], + baselineExclusions: [ + { + version: 1, + agent: "openclaw", + key: "phone_home", + digest: DIGEST, + acknowledgedAt: "2026-01-01T00:00:00.000Z", + }, + ], + sandboxGpuProof: { + status: "verified", + cudaVerified: true, + label: "cuda", + detail: "first diagnostic", + at: "2026-01-01T00:00:00.000Z", + }, + }; + const second: SandboxEntry = { + ...first, + createdAt: "2026-06-01T00:00:00.000Z", + customPolicies: [ + { + ...first.customPolicies?.[0], + name: "custom", + content: POLICY_A, + sourcePath: "/second/policy.yaml", + appliedAt: "2026-06-01T00:00:00.000Z", + }, + ], + baselineExclusions: [ + { + ...first.baselineExclusions?.[0], + version: 1, + agent: "openclaw", + key: "phone_home", + digest: DIGEST, + acknowledgedAt: "2026-06-01T00:00:00.000Z", + }, + ], + sandboxGpuProof: { + ...first.sandboxGpuProof!, + detail: "second diagnostic", + at: "2026-06-01T00:00:00.000Z", + }, + }; + expect(launchReadinessDigest(buildLaunchReadinessRegistryProjection(second, agent))).toBe( + launchReadinessDigest(buildLaunchReadinessRegistryProjection(first, agent)), + ); + }); + + it("distinguishes authoritative final validation failure from evidence failure", async () => { + const first = await inspectLaunchReadiness(SANDBOX, deps()); + const publication = publicationFromDecision(SANDBOX, first); + + const invalid = deps(); + invalid.gatewayHealth = async () => false; + expect(await publishLaunchReadiness(publication, invalid)).toEqual({ + kind: "validation-failed", + category: "health", + }); + + const changedRoute = deps(); + changedRoute.capture = (args) => ({ + status: 0, + output: + args[0] === "policy" + ? policy + : "Gateway Inference:\n\n Provider: nvidia\n Model: changed\n", + stdout: + args[0] === "policy" + ? policy + : "Gateway Inference:\n\n Provider: nvidia\n Model: changed\n", + stderr: "", + }); + expect(await publishLaunchReadiness(publication, changedRoute)).toEqual({ + kind: "validation-failed", + category: "config", + }); + + const observationUnavailable = deps(); + observationUnavailable.observeSandbox = () => { + throw new Error("observer unavailable"); + }; + expect(await publishLaunchReadiness(publication, observationUnavailable)).toEqual({ + kind: "evidence-failed", + }); + + const hashUnavailable = deps(); + hashUnavailable.capture = (args) => ({ + status: 0, + output: args[0] === "policy" ? "version: [" : routeOutput, + stdout: args[0] === "policy" ? "version: [" : routeOutput, + stderr: "", + }); + expect(await publishLaunchReadiness(publication, hashUnavailable)).toEqual({ + kind: "evidence-failed", + }); + + const inferenceObservationUnavailable = deps(); + inferenceObservationUnavailable.capture = (args) => ({ + status: 0, + output: args[0] === "policy" ? policy : "unexpected inference output", + stdout: args[0] === "policy" ? policy : "unexpected inference output", + stderr: "", + }); + expect(await publishLaunchReadiness(publication, inferenceObservationUnavailable)).toEqual({ + kind: "evidence-failed", + }); + + const unavailable = deps(); + unavailable.publishLease = () => { + throw new Error("unavailable"); + }; + expect(await publishLaunchReadiness(publication, unavailable)).toEqual({ + kind: "evidence-failed", + }); + }); + + it("never validates or publishes evidence without a fenced epoch (#8942)", async () => { + const currentDeps = deps(); + const publishLease = vi.fn(); + currentDeps.publishLease = publishLease; + + await expect( + publishLaunchReadiness( + { + sandboxName: SANDBOX, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + epochId: null, + }, + currentDeps, + ), + ).resolves.toEqual({ kind: "evidence-failed" }); + expect(publishLease).not.toHaveBeenCalled(); + }); + + it("rejects in-progress lifecycle and policy mutations", () => { + const agent = loadAgent("openclaw"); + expect(() => + buildLaunchReadinessRegistryProjection( + { ...sandbox, pendingRouteReservation: true, reservationSessionId: "session" }, + agent, + ), + ).toThrow(); + expect(() => + buildLaunchReadinessRegistryProjection( + { + ...sandbox, + baselineExclusionTransition: { + id: "transition", + operation: "exclude", + exclusion: { + version: 1, + agent: "openclaw", + key: "phone_home", + digest: DIGEST, + }, + targetLiveDigest: null, + startedAt: "2026-01-01T00:00:00.000Z", + }, + }, + agent, + ), + ).toThrow(); + }); +}); diff --git a/src/lib/actions/sandbox/launch-readiness.ts b/src/lib/actions/sandbox/launch-readiness.ts new file mode 100644 index 00000000000..1b4a2e38872 --- /dev/null +++ b/src/lib/actions/sandbox/launch-readiness.ts @@ -0,0 +1,949 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; + +import type { AgentDefinition } from "../../agent/defs"; +import { log } from "../../cli/logger"; +import { getCuaRuntimeReadinessDigest } from "../../cua/contract"; +import { parseGatewayInference, planInferenceRouteReconcile } from "../../inference/config"; +import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; +import { normalizeInferenceSelection } from "../../inference/selection"; +import { parseServingProfileProvenance } from "../../inference/serving/profile-provenance"; +import { resolveGatewayName } from "../../onboard/gateway-binding"; +import { + observeSandboxOnGateway, + type SandboxRecreateObserver, +} from "../../onboard/sandbox-recreate-probe"; +import { assertNoOpenShellGatewayEndpointOverride } from "../../openshell-gateway-endpoint-guard"; +import { parseAndValidateSandboxPolicy } from "../../policy/sandbox-policy-validation"; +import { + checkLaunchReadinessMutationAuthority, + fenceLaunchReadinessLease, + type LaunchReadinessFence, + LaunchReadinessFenceError, + type LaunchReadinessIdentity, + type LaunchReadinessLeaseRead, + type LaunchReadinessStoreOptions, + publishLaunchReadinessLease, + readLaunchReadinessLease, +} from "../../state/launch-readiness-lease"; +import { withMcpLifecycleLock as withSandboxMutationLock } from "../../state/mcp-lifecycle-lock-acquisition"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { normalizeSandboxMcpState } from "../../state/registry-mcp"; +import { + cloneSandboxMessagingState, + serializeSandboxMessagingStateForDisk, +} from "../../state/registry-messaging"; +import { buildGatewayInferenceGetArgs } from "./connect-inference-gateway"; +import { + captureLaunchReadiness, + LaunchReadinessEvidenceError, + type LaunchReadinessHealthDeps, + LaunchReadinessObservationError as ObservationError, + requireLaunchSemanticHealth, + resolveLaunchInteractiveCommand, + resolveTrustedLaunchAgent, +} from "./launch-readiness/health"; + +const LIVE_POLICY_MAX_BYTES = 2 * 1_024 * 1_024; +const ALLOWED_OPENSHELL_DRIVERS = new Set(["docker", "kubernetes", "vm"]); + +export type LaunchReadinessPerformanceStage = + | "storage-read" + | "live-validation" + | "evidence-fence" + | "publication-validation" + | "publication-store"; + +export type LaunchReadinessDecisionCategory = + | "accepted" + | "missing" + | "unsafe" + | "malformed" + | "expired" + | "identity" + | "config" + | "health" + | "session"; + +export type LaunchReadinessDecision = + | { + kind: "accepted"; + category: "accepted"; + agent: AgentDefinition; + sb: SandboxEntry; + } + | { + kind: "fallback"; + category: Exclude; + fence: LaunchReadinessFence | null; + gatewayName: string | null; + gatewayPort: number | null; + fenceFailed: boolean; + recoveryBlocked: boolean; + authorityUnsupported?: true; + }; + +export interface LaunchReadinessDeps extends LaunchReadinessHealthDeps { + checkMutationAuthority?: typeof checkLaunchReadinessMutationAuthority; + getSandbox?: typeof registry.getSandbox; + observeSandbox?: SandboxRecreateObserver; + readLease?: typeof readLaunchReadinessLease; + fenceLease?: typeof fenceLaunchReadinessLease; + publishLease?: typeof publishLaunchReadinessLease; + storeOptions?: LaunchReadinessStoreOptions; + withSandboxLock?: typeof withSandboxMutationLock; + withGatewayLock?: typeof withGatewayRouteMutationLock; +} + +export interface LaunchReadinessPublication { + sandboxName: string; + gatewayName: string | null; + gatewayPort: number | null; + epochId: string | null; +} + +export type LaunchReadinessPublicationResult = + | { kind: "published" } + | { + kind: "validation-failed"; + category: "identity" | "config" | "health" | "session"; + } + | { kind: "evidence-failed" }; + +export type LaunchReadinessMutationGateResult = + | { kind: "entered"; value: T } + | { kind: "changed" } + | { kind: "unsafe" }; + +type LaunchReadinessPublicationValidationCategory = Extract< + LaunchReadinessPublicationResult, + { kind: "validation-failed" } +>["category"]; + +function recordPerformanceStage(stage: LaunchReadinessPerformanceStage, startedAt: number): void { + try { + performance.measure(`nemoclaw.launch-readiness.${stage}`, { + start: startedAt, + end: performance.now(), + }); + } catch { + // Measurements are diagnostic evidence and never control launch behavior. + } +} + +function normalizedString(value: unknown): string | null { + if (typeof value !== "string") return null; + const normalized = value.trim(); + return normalized.length > 0 ? normalized : null; +} + +function exactNonemptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function canonicalize(value: unknown): unknown { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) + ) { + return value; + } + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalize(entry)]), + ); + } + throw new ObservationError("config"); +} + +export function launchReadinessDigest(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(canonicalize(value))) + .digest("hex"); +} + +function exactContentDigest(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function projectHostMounts(entry: SandboxEntry): unknown[] { + return (entry.hostMounts ?? []).map((mount) => { + const source = exactNonemptyString(mount.source); + const target = exactNonemptyString(mount.target); + const device = exactNonemptyString(mount.sourceIdentity?.device); + const inode = exactNonemptyString(mount.sourceIdentity?.inode); + if (!source || !target || mount.readOnly !== true || !device || !inode) { + throw new ObservationError("config"); + } + return { + sourceSha256: exactContentDigest(source), + target, + readOnly: true, + sourceIdentity: { device, inode }, + }; + }); +} + +function projectServingProfile(entry: SandboxEntry): unknown { + if (entry.servingProfileProvenance === undefined) return null; + const provenance = parseServingProfileProvenance(entry.servingProfileProvenance); + if (!provenance) throw new ObservationError("config"); + return { + schemaVersion: provenance.schemaVersion, + catalogDigest: provenance.catalogDigest, + preset: { + id: provenance.preset.id, + digest: provenance.preset.digest, + displayName: provenance.preset.displayName, + supportState: provenance.preset.supportState, + }, + recipe: { + id: provenance.recipe.id, + digest: provenance.recipe.digest, + backend: provenance.recipe.backend, + }, + model: { + id: provenance.model.id, + revision: provenance.model.revision, + }, + runtimeImage: provenance.runtimeImage, + estimatedImageDownloadBytes: provenance.estimatedImageDownloadBytes, + estimatedModelDownloadBytes: provenance.estimatedModelDownloadBytes, + }; +} + +function projectOptionalBoolean(value: unknown): boolean { + if (value === undefined) return false; + if (typeof value !== "boolean") throw new ObservationError("config"); + return value; +} + +export function launchReadinessPolicyDigest(content: string): string { + return launchReadinessDigest(parseAndValidateSandboxPolicy(content)); +} + +function projectWorkload(workload: SandboxWorkloadReceipt | undefined): unknown { + if (!workload) return null; + if (workload.kind === "legacy-dockerfile") { + return { + schemaVersion: workload.schemaVersion, + kind: workload.kind, + reference: workload.reference, + shared: workload.shared, + }; + } + if (workload.kind === "managed-image") { + return { + schemaVersion: workload.schemaVersion, + kind: workload.kind, + reference: workload.reference, + platform: workload.platform ?? null, + release: workload.release, + sourceRevision: workload.sourceRevision, + sourceCohort: workload.sourceCohort, + capabilityContractVersion: workload.capabilityContractVersion, + startupProfileContractVersion: workload.startupProfileContractVersion, + startupProfileSha256: workload.startupProfileSha256, + credentialProxyReplayRequired: workload.credentialProxyReplayRequired, + corporateCaSha256: workload.corporateCaB64 + ? exactContentDigest(workload.corporateCaB64) + : null, + shared: workload.shared, + }; + } + return { + schemaVersion: workload.schemaVersion, + kind: workload.kind, + contractVersion: workload.contractVersion, + agent: workload.agent, + platform: workload.platform, + artifact: { + digest: workload.artifact.digest, + version: workload.artifact.version, + sourceRepository: workload.artifact.source.repository, + sourceRevision: workload.artifact.source.revision, + }, + launch: { + executableRelativePath: workload.launch.executable.relativePath, + executableDigest: workload.launch.executable.digest, + arguments: [...workload.launch.arguments], + workingDirectory: workload.launch.workingDirectory, + environmentNames: [...workload.launch.environmentNames], + }, + startupProfileContractVersion: workload.startupProfileContractVersion, + startupProfileSha256: workload.startupProfileSha256, + credentialProxyReplayRequired: workload.credentialProxyReplayRequired, + shared: workload.shared, + }; +} + +function projectMcpState(value: unknown): unknown { + const state = normalizeSandboxMcpState(value); + if (!state) return null; + if (state.destroyPreparedAt || state.destroyPendingAt) throw new ObservationError("config"); + return { + bridges: Object.values(state.bridges) + .map((bridge) => { + if (bridge.addState) throw new ObservationError("config"); + const endpoint = new URL(bridge.url); + if (endpoint.username || endpoint.password || endpoint.search || endpoint.hash) { + throw new ObservationError("config"); + } + return { + server: bridge.server, + agent: bridge.agent, + adapter: bridge.adapter ?? null, + url: bridge.url, + env: [...bridge.env], + trustedPrivateHost: bridge.trustedPrivateHost ?? null, + allowedIps: bridge.allowedIps ? [...bridge.allowedIps] : null, + providerName: bridge.providerName ?? null, + providerId: bridge.providerId ?? null, + policyName: bridge.policyName, + }; + }) + .sort((left, right) => left.server.localeCompare(right.server)), + managedServerNames: [...(state.managedServerNames ?? [])].sort(), + }; +} + +function projectMessagingState(entry: SandboxEntry): unknown { + const state = cloneSandboxMessagingState(entry.messaging); + const persisted = serializeSandboxMessagingStateForDisk(entry.messaging); + if (!state || !persisted) return null; + const originalChannels = new Map( + state.plan.channels.map((channel) => [channel.channelId, channel]), + ); + return { + schemaVersion: persisted.schemaVersion, + plan: { + schemaVersion: persisted.plan.schemaVersion, + sandboxName: persisted.plan.sandboxName, + agent: persisted.plan.agent, + workflow: persisted.plan.workflow, + disabledChannels: [...persisted.plan.disabledChannels], + networkPolicy: persisted.plan.networkPolicy, + channels: persisted.plan.channels.map((channel) => { + const originalInputs = new Map( + (originalChannels.get(channel.channelId)?.inputs ?? []).map((input) => [ + input.inputId, + input, + ]), + ); + return { + channelId: channel.channelId, + active: channel.active ?? null, + configured: channel.configured, + disabled: channel.disabled, + inputs: (channel.inputs ?? []).map((input) => ({ + inputId: input.inputId, + credentialAvailable: input.credentialAvailable ?? null, + value: + originalInputs.get(input.inputId)?.kind === "config" ? (input.value ?? null) : null, + })), + hooks: channel.hooks ?? [], + }; + }), + credentialBindings: (persisted.plan.credentialBindings ?? []).map((binding) => ({ + channelId: binding.channelId, + providerEnvKey: binding.providerEnvKey, + credentialAvailable: binding.credentialAvailable, + })), + }, + }; +} + +function projectAgent(agent: AgentDefinition): unknown { + let manifestSha256: string; + try { + manifestSha256 = exactContentDigest(fs.readFileSync(agent.manifestPath, "utf8")); + } catch { + throw new LaunchReadinessEvidenceError(); + } + return { + version: 1, + manifestSha256, + name: agent.name, + binaryPath: agent.binary_path ?? null, + versionCommand: agent.versionCommand, + expectedVersion: agent.expectedVersion, + versionScheme: agent.versionScheme ?? null, + gatewayCommand: agent.gateway_command ?? null, + runtime: { + kind: agent.runtime?.kind ?? "gateway", + interactiveCommand: agent.runtime?.interactive_command ?? null, + headlessCommand: agent.runtime?.headless_command ?? null, + smokeCommands: [...(agent.runtime?.smoke_commands ?? [])], + }, + forwardPorts: [...(agent.forward_ports ?? [])], + devicePairing: agent.hasDevicePairing, + phoneHomeHosts: [...agent.phoneHomeHosts], + healthProbe: agent.healthProbe + ? { + url: agent.healthProbe.url, + port: agent.healthProbe.port, + timeoutSeconds: agent.healthProbe.timeout_seconds, + } + : null, + dashboard: { + kind: agent.dashboard.kind, + path: agent.dashboard.path, + healthPath: agent.dashboard.healthPath, + auth: agent.dashboard.auth, + }, + webAuth: { + method: agent.webAuth.method, + env: agent.webAuth.env, + }, + dashboardUi: agent.dashboardUi + ? { + label: agent.dashboardUi.label, + port: agent.dashboardUi.port, + path: agent.dashboardUi.path, + enableEnv: agent.dashboardUi.enableEnv, + portEnv: agent.dashboardUi.portEnv, + tuiEnv: agent.dashboardUi.tuiEnv, + } + : null, + configPaths: { + dir: agent.configPaths.dir, + configFile: agent.configPaths.configFile, + envFile: agent.configPaths.envFile, + format: agent.configPaths.format, + shieldsFiles: [...agent.configPaths.shieldsFiles], + }, + inference: { + providerType: agent.inference?.provider_type ?? null, + providerOptions: [...agent.inferenceProviderOptions], + defaultModel: agent.inference?.default_model ?? null, + }, + mcp: { + support: agent.mcpCapability.support, + adapter: agent.mcpCapability.adapter ?? null, + reason: agent.mcpCapability.reason ?? null, + }, + stateLockPlan: { + version: agent.stateLockPlan.version, + readOnlyRoots: [...agent.stateLockPlan.readOnlyRoots], + confidentialRoots: [...agent.stateLockPlan.confidentialRoots], + readOnlyPrefixes: [...agent.stateLockPlan.readOnlyPrefixes], + confidentialPrefixes: [...agent.stateLockPlan.confidentialPrefixes], + writableSubpaths: [...agent.stateLockPlan.writableSubpaths], + }, + stateLockPlanInImage: agent.stateLockPlanInImage, + }; +} + +export function buildLaunchReadinessRegistryProjection( + entry: SandboxEntry, + agent: AgentDefinition, +): unknown { + const driver = normalizedString(entry.openshellDriver)?.toLowerCase() ?? null; + if (!driver || !ALLOWED_OPENSHELL_DRIVERS.has(driver)) throw new ObservationError("config"); + const openshellVersion = normalizedString(entry.openshellVersion); + const gatewayPort = entry.gatewayPort; + if (!openshellVersion || openshellVersion.length > 128) throw new ObservationError("config"); + if (!Number.isInteger(gatewayPort) || (gatewayPort ?? 0) < 1 || (gatewayPort ?? 0) > 65535) { + throw new ObservationError("config"); + } + const gatewayName = resolveGatewayName(gatewayPort as number); + if (entry.gatewayName !== gatewayName) throw new ObservationError("config"); + const lifecycleGeneration = normalizedString(entry.lifecycleGeneration); + const liveIdentityFingerprint = normalizedString(entry.lifecycleLiveIdentityFingerprint); + if (!lifecycleGeneration || !liveIdentityFingerprint) throw new ObservationError("identity"); + if (entry.pendingRouteReservation === true || entry.reservationSessionId) { + throw new ObservationError("config"); + } + if (entry.baselineExclusionTransition) throw new ObservationError("config"); + + const customPolicies = (entry.customPolicies ?? []).map((policy) => ({ + name: policy.name, + contentSha256: exactContentDigest(policy.content), + pendingContentSha256: + typeof policy.pendingContent === "string" ? exactContentDigest(policy.pendingContent) : null, + pinAuthoritySha256: policy.trustedPrivatePins + ? launchReadinessDigest({ + version: policy.trustedPrivatePins.version, + contentDigest: policy.trustedPrivatePins.contentDigest, + }) + : null, + })); + const baselineExclusions = (entry.baselineExclusions ?? []).map((exclusion) => ({ + version: exclusion.version, + agent: exclusion.agent, + key: exclusion.key, + digest: exclusion.digest, + appliedAgentVersion: exclusion.appliedAgentVersion ?? null, + })); + const inference = normalizeInferenceSelection(entry); + if ( + inference.credentialEnv !== null && + !/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(inference.credentialEnv) + ) { + throw new ObservationError("config"); + } + if (inference.endpointUrl !== null) { + let endpoint: URL; + try { + endpoint = new URL(inference.endpointUrl); + } catch { + throw new ObservationError("config"); + } + if ( + !["http:", "https:"].includes(endpoint.protocol) || + endpoint.username || + endpoint.password || + endpoint.search || + endpoint.hash + ) { + throw new ObservationError("config"); + } + } + const agentName = normalizedString(entry.agent) ?? "openclaw"; + const interactiveCommand = resolveLaunchInteractiveCommand(agent, agentName); + const sandboxGpuMode = entry.sandboxGpuMode ?? null; + const sandboxGpuDevice = entry.sandboxGpuDevice ?? null; + if (sandboxGpuMode !== null && typeof sandboxGpuMode !== "string") { + throw new ObservationError("config"); + } + if (sandboxGpuDevice !== null && typeof sandboxGpuDevice !== "string") { + throw new ObservationError("config"); + } + const hermesAuthMethod = entry.hermesAuthMethod ?? null; + if (hermesAuthMethod !== null && hermesAuthMethod !== "oauth" && hermesAuthMethod !== "api_key") { + throw new ObservationError("config"); + } + + return { + version: 2, + name: entry.name, + openshellDriver: driver, + openshellVersion, + gatewayName, + gatewayPort, + lifecycleGeneration, + lifecycleLiveIdentityFingerprint: liveIdentityFingerprint, + agent: agentName, + agentVersion: normalizedString(entry.agentVersion), + nemoclawVersion: normalizedString(entry.nemoclawVersion), + imageTag: normalizedString(entry.imageTag), + workloadIdentitySha256: launchReadinessDigest(projectWorkload(entry.workload)), + fromDockerfile: normalizedString(entry.fromDockerfile), + servingProfileProvenance: projectServingProfile(entry), + hostMounts: projectHostMounts(entry), + gpuEnabled: projectOptionalBoolean(entry.gpuEnabled), + hostGpuDetected: projectOptionalBoolean(entry.hostGpuDetected), + sandboxGpuEnabled: projectOptionalBoolean(entry.sandboxGpuEnabled), + sandboxGpuMode, + sandboxGpuDevice, + interactiveCommand, + sandboxGpuProof: entry.sandboxGpuProof + ? { + status: entry.sandboxGpuProof.status, + cudaVerified: entry.sandboxGpuProof.cudaVerified, + label: entry.sandboxGpuProof.label ?? null, + } + : null, + inference, + policies: [...(entry.policies ?? [])], + policyTier: normalizedString(entry.policyTier), + policyPresetsFinalized: entry.policyPresetsFinalized === true, + customPolicies, + baselineExclusions, + webSearchEnabled: entry.webSearchEnabled === true, + webSearchProvider: entry.webSearchProvider ?? null, + toolDisclosure: entry.toolDisclosure ?? null, + observabilityEnabled: entry.observabilityEnabled === true, + dcodeAutoApprovalMode: entry.dcodeAutoApprovalMode ?? null, + messagingSha256: launchReadinessDigest(projectMessagingState(entry)), + mcpSha256: launchReadinessDigest(projectMcpState(entry.mcp)), + hermesToolGateways: [...(entry.hermesToolGateways ?? [])], + hermesInferenceProvider: normalizedString(entry.hermesInferenceProvider), + hermesAuthMethod, + hermesDashboardEnabled: entry.hermesDashboardEnabled === true, + hermesDashboardPort: entry.hermesDashboardPort ?? null, + hermesDashboardInternalPort: entry.hermesDashboardInternalPort ?? null, + hermesDashboardTui: entry.hermesDashboardTui === true, + dashboardPort: entry.dashboardPort ?? null, + dashboardRemoteBindPrepared: entry.dashboardRemoteBindPrepared === true, + openclawImagePluginInstalls: (entry.openclawImagePluginInstalls ?? []).map((install) => ({ + id: install.id, + installPath: install.installPath, + loadPaths: install.loadPaths ? [...install.loadPaths] : null, + })), + cuaRuntimeReadinessSha256: entry.cuaRuntimeReadiness + ? launchReadinessDigest(getCuaRuntimeReadinessDigest(entry.cuaRuntimeReadiness)) + : null, + }; +} + +function classifyReceipt( + read: LaunchReadinessLeaseRead, +): Exclude { + return read.kind === "valid" ? "config" : read.kind; +} + +function captureLivePolicy( + sandboxName: string, + gatewayName: string, + deps: LaunchReadinessDeps, +): string { + const result = ( + deps.capture ?? ((args) => captureLaunchReadiness(args, { maxBuffer: LIVE_POLICY_MAX_BYTES })) + )(["policy", "get", "-g", gatewayName, "--full", sandboxName]); + if (result.status !== 0 || !result.output?.trim()) throw new LaunchReadinessEvidenceError(); + try { + return launchReadinessPolicyDigest(result.output); + } catch { + throw new LaunchReadinessEvidenceError(); + } +} + +function reportsInferenceNotConfigured(output: string): boolean { + const lines = output.replace(/\u001b\[[0-9;]*m/g, "").split("\n"); + let inGatewayInference = false; + for (const line of lines) { + if (/^(?:Gateway )?Inference:\s*$/i.test(line)) { + inGatewayInference = true; + continue; + } + if (inGatewayInference && /^\S.*:$/.test(line)) return false; + if (inGatewayInference && /^Not configured$/i.test(line.trim())) return true; + } + return false; +} + +async function captureLaunchIdentity( + sandboxName: string, + gatewayName: string, + gatewayPort: number, + deps: LaunchReadinessDeps, +): Promise<{ identity: LaunchReadinessIdentity; agent: AgentDefinition; sb: SandboxEntry }> { + try { + assertNoOpenShellGatewayEndpointOverride(); + } catch { + throw new ObservationError("config"); + } + const getSandbox = deps.getSandbox ?? registry.getSandbox; + const entry = getSandbox(sandboxName); + if (!entry || entry.name !== sandboxName) throw new ObservationError("identity"); + const agentName = normalizedString(entry.agent) ?? "openclaw"; + const agent = resolveTrustedLaunchAgent(entry, deps, agentName); + const projection = buildLaunchReadinessRegistryProjection(entry, agent); + if (entry.gatewayPort !== gatewayPort || entry.gatewayName !== gatewayName) { + throw new ObservationError("identity"); + } + const lifecycleGeneration = normalizedString(entry.lifecycleGeneration); + const recordedFingerprint = normalizedString(entry.lifecycleLiveIdentityFingerprint); + if (!lifecycleGeneration || !recordedFingerprint) throw new ObservationError("identity"); + + let live: ReturnType; + try { + live = (deps.observeSandbox ?? observeSandboxOnGateway)({ + sandboxName, + gatewayName, + gatewayPort, + }); + } catch { + throw new LaunchReadinessEvidenceError(); + } + if (live.state === "missing") throw new ObservationError("identity"); + if (live.state !== "ready") throw new ObservationError("health"); + if (live.liveIdentityFingerprint !== recordedFingerprint) { + throw new ObservationError("identity"); + } + + const livePolicy = captureLivePolicy(sandboxName, gatewayName, deps); + const inferenceSelection = normalizeInferenceSelection(entry); + const inference = registry.getSandboxEntryInference(entry); + const inferenceResult = (deps.capture ?? ((args) => captureLaunchReadiness(args)))( + buildGatewayInferenceGetArgs(gatewayName), + ); + if (inferenceResult.status !== 0) throw new LaunchReadinessEvidenceError(); + const liveInference = parseGatewayInference(inferenceResult.output); + const liveInferenceAbsent = reportsInferenceNotConfigured(inferenceResult.output); + if (inference.kind === "configured") { + if (!liveInference && !liveInferenceAbsent) throw new LaunchReadinessEvidenceError(); + if (planInferenceRouteReconcile(liveInference, inference).kind !== "aligned") { + throw new ObservationError("config"); + } + } else { + if (liveInference) throw new ObservationError("config"); + if (!liveInferenceAbsent) throw new LaunchReadinessEvidenceError(); + } + + await requireLaunchSemanticHealth( + sandboxName, + gatewayName, + agentName, + entry, + agent, + inference.kind === "configured", + deps, + ); + + return { + identity: { + registry: launchReadinessDigest(projection), + agent: launchReadinessDigest(projectAgent(agent)), + livePolicy, + liveInference: launchReadinessDigest({ + selection: inferenceSelection, + live: liveInference + ? { provider: liveInference.provider, model: liveInference.model } + : null, + }), + gatewayName, + lifecycleGeneration, + liveIdentityFingerprint: recordedFingerprint, + }, + agent, + sb: entry, + }; +} + +function identityMatches(left: LaunchReadinessIdentity, right: LaunchReadinessIdentity): boolean { + return ( + left.registry === right.registry && + left.agent === right.agent && + left.livePolicy === right.livePolicy && + left.liveInference === right.liveInference && + left.gatewayName === right.gatewayName && + left.lifecycleGeneration === right.lifecycleGeneration && + left.liveIdentityFingerprint === right.liveIdentityFingerprint + ); +} + +function debugDecision(category: LaunchReadinessDecisionCategory): void { + log.debug( + category === "accepted" + ? "Launch readiness: accepted" + : `Launch readiness: fallback due to ${category}`, + ); +} + +function publicationValidationCategory( + error: unknown, +): LaunchReadinessPublicationValidationCategory | null { + if (!(error instanceof ObservationError)) return null; + return ["identity", "config", "health", "session"].includes(error.category) + ? (error.category as LaunchReadinessPublicationValidationCategory) + : null; +} + +function fallback( + category: Exclude, + fence: LaunchReadinessFence | null, + gatewayName: string | null, + gatewayPort: number | null, + fenceFailed: boolean, + recoveryBlocked = false, + authorityUnsupported = false, +): LaunchReadinessDecision { + debugDecision(category); + return { + kind: "fallback", + category, + fence, + gatewayName, + gatewayPort, + fenceFailed, + recoveryBlocked, + ...(authorityUnsupported ? { authorityUnsupported: true as const } : {}), + }; +} + +/** + * Validate or fence launch evidence under the canonical lifecycle then route + * lock order. No recovery or readiness polling may run inside this function. + */ +export async function inspectLaunchReadiness( + sandboxName: string, + deps: LaunchReadinessDeps = {}, +): Promise { + const withSandboxLock = deps.withSandboxLock ?? withSandboxMutationLock; + const withGatewayLock = deps.withGatewayLock ?? withGatewayRouteMutationLock; + return withSandboxLock(sandboxName, async () => { + const entry = (deps.getSandbox ?? registry.getSandbox)(sandboxName); + if (!entry) return fallback("missing", null, null, null, true, true); + let gatewayPort: number; + let gatewayName: string; + try { + gatewayPort = entry.gatewayPort as number; + if (!Number.isInteger(gatewayPort) || gatewayPort < 1 || gatewayPort > 65535) { + throw new ObservationError("config"); + } + gatewayName = resolveGatewayName(gatewayPort); + if (entry.gatewayName !== gatewayName) throw new ObservationError("config"); + } catch (error) { + const category = error instanceof ObservationError ? error.category : "config"; + return fallback(category, null, null, null, true, true); + } + return withGatewayLock(gatewayName, async () => { + const storageStartedAt = performance.now(); + const read = (deps.readLease ?? readLaunchReadinessLease)( + sandboxName, + gatewayName, + gatewayPort, + deps.storeOptions, + ); + recordPerformanceStage("storage-read", storageStartedAt); + let category: Exclude = classifyReceipt(read); + if (read.kind === "valid") { + const validationStartedAt = performance.now(); + try { + const captured = await captureLaunchIdentity(sandboxName, gatewayName, gatewayPort, deps); + if (identityMatches(read.lease.identity, captured.identity)) { + debugDecision("accepted"); + return { + kind: "accepted", + category: "accepted", + agent: captured.agent, + sb: captured.sb, + }; + } + category = "config"; + } catch (error) { + category = error instanceof ObservationError ? error.category : "unsafe"; + } finally { + recordPerformanceStage("live-validation", validationStartedAt); + } + } + + const fenceStartedAt = performance.now(); + try { + const fence = (deps.fenceLease ?? fenceLaunchReadinessLease)( + sandboxName, + gatewayName, + gatewayPort, + deps.storeOptions, + ); + return fallback(category, fence, gatewayName, gatewayPort, false); + } catch (error) { + const recoveryBlocked = + error instanceof LaunchReadinessFenceError ? error.blocksRecovery : true; + return fallback( + category === "missing" ? "unsafe" : category, + null, + gatewayName, + gatewayPort, + true, + recoveryBlocked, + error instanceof LaunchReadinessFenceError && error.authorityUnsupported, + ); + } finally { + recordPerformanceStage("evidence-fence", fenceStartedAt); + } + }); + }); +} + +/** + * Enter the complete preflight mutation window only while the producer's + * runtime epoch remains authoritative. Both canonical locks stay held until + * the operation, final recapture, and publication finish. Nested lifecycle or + * gateway lock users are reentrant through the canonical lock implementation. + */ +export async function withLaunchReadinessMutationGate( + publication: LaunchReadinessPublication, + operation: () => Promise | T, + deps: LaunchReadinessDeps = {}, +): Promise> { + const { sandboxName, gatewayName, gatewayPort, epochId } = publication; + if (!gatewayName || !gatewayPort) return { kind: "unsafe" }; + const withSandboxLock = deps.withSandboxLock ?? withSandboxMutationLock; + const withGatewayLock = deps.withGatewayLock ?? withGatewayRouteMutationLock; + return withSandboxLock(sandboxName, async () => { + const entry = (deps.getSandbox ?? registry.getSandbox)(sandboxName); + if (entry?.gatewayPort !== gatewayPort || entry.gatewayName !== gatewayName) { + return { kind: "changed" }; + } + return withGatewayLock(gatewayName, async () => { + const authority = (deps.checkMutationAuthority ?? checkLaunchReadinessMutationAuthority)( + sandboxName, + gatewayName, + gatewayPort, + epochId, + deps.storeOptions, + ); + if (authority !== "current") return { kind: authority }; + return { kind: "entered", value: await operation() }; + }); + }); +} + +/** Re-observe final state and publish only when the original fence still wins. */ +export async function publishLaunchReadiness( + publication: LaunchReadinessPublication, + deps: LaunchReadinessDeps = {}, +): Promise { + const { sandboxName, gatewayName, gatewayPort, epochId } = publication; + if (!gatewayName || !gatewayPort || !epochId) return { kind: "evidence-failed" }; + const withSandboxLock = deps.withSandboxLock ?? withSandboxMutationLock; + const withGatewayLock = deps.withGatewayLock ?? withGatewayRouteMutationLock; + try { + return await withSandboxLock(sandboxName, async () => { + const entry = (deps.getSandbox ?? registry.getSandbox)(sandboxName); + if (entry?.gatewayPort !== gatewayPort || entry.gatewayName !== gatewayName) { + return { kind: "validation-failed", category: "identity" } as const; + } + return withGatewayLock(gatewayName, async () => { + const validationStartedAt = performance.now(); + let captured: Awaited>; + try { + captured = await captureLaunchIdentity(sandboxName, gatewayName, gatewayPort, deps); + } catch (error) { + const category = publicationValidationCategory(error); + return category + ? ({ kind: "validation-failed", category } as const) + : ({ kind: "evidence-failed" } as const); + } finally { + recordPerformanceStage("publication-validation", validationStartedAt); + } + const publicationStartedAt = performance.now(); + try { + (deps.publishLease ?? publishLaunchReadinessLease)( + sandboxName, + gatewayName, + gatewayPort, + epochId, + captured.identity, + deps.storeOptions, + ); + } catch { + return { kind: "evidence-failed" } as const; + } finally { + recordPerformanceStage("publication-store", publicationStartedAt); + } + return { kind: "published" } as const; + }); + }); + } catch { + return { kind: "evidence-failed" }; + } +} + +export function publicationFromDecision( + sandboxName: string, + decision: LaunchReadinessDecision, +): LaunchReadinessPublication { + if (decision.kind === "accepted") { + return { sandboxName, gatewayName: null, gatewayPort: null, epochId: null }; + } + return { + sandboxName, + gatewayName: decision.gatewayName, + gatewayPort: decision.gatewayPort, + epochId: decision.fence?.epochId ?? null, + }; +} diff --git a/src/lib/actions/sandbox/launch-readiness/health.ts b/src/lib/actions/sandbox/launch-readiness/health.ts new file mode 100644 index 00000000000..ac8dd7f08c6 --- /dev/null +++ b/src/lib/actions/sandbox/launch-readiness/health.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { captureOpenshell } from "../../../adapters/openshell/runtime"; +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../../adapters/openshell/timeouts"; +import type { AgentDefinition } from "../../../agent/defs"; +import { isTerminalAgent, listAgents, loadAgent } from "../../../agent/defs"; +import * as agentRuntime from "../../../agent/runtime"; +import { runAgentSmokeCommands } from "../../../agent/terminal-smoke"; +import { requireCuaLifecycleReadiness } from "../../../cua/lifecycle-readiness"; +import type { SandboxEntry } from "../../../state/registry"; +import { + buildSandboxInferenceRouteProbeArgs, + type InferenceRouteProbeAgent, + parseSandboxInferenceRouteProbeResult, +} from "../connect-inference-route-probe"; +import { areSandboxLaunchForwardsHealthy } from "../forward-recovery"; +import { isSandboxGatewayRunningForStatus } from "../process-recovery"; + +export type LaunchReadinessObservationCategory = + | "missing" + | "unsafe" + | "malformed" + | "expired" + | "identity" + | "config" + | "health" + | "session"; + +export type LaunchReadinessCaptureResult = ReturnType; + +export interface LaunchReadinessHealthDeps { + listAgents?: typeof listAgents; + loadAgent?: typeof loadAgent; + capture?: (args: string[]) => LaunchReadinessCaptureResult; + gatewayHealth?: (sandboxName: string, gatewayName: string) => Promise; + forwardsHealthy?: (sandboxName: string, gatewayName: string) => boolean | null; + smoke?: typeof runAgentSmokeCommands; + inferenceProbe?: ( + sandboxName: string, + agent: InferenceRouteProbeAgent, + gatewayName: string, + ) => ReturnType; + cuaReadiness?: typeof requireCuaLifecycleReadiness; +} + +export class LaunchReadinessObservationError extends Error { + constructor(readonly category: LaunchReadinessObservationCategory) { + super(category); + } +} + +/** The required observation could not produce authoritative evidence. */ +export class LaunchReadinessEvidenceError extends Error { + constructor() { + super("launch readiness evidence unavailable"); + } +} + +export function captureLaunchReadiness( + args: string[], + options: { includeStreams?: boolean; maxBuffer?: number } = {}, +): LaunchReadinessCaptureResult { + return captureOpenshell(args, { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + ...options, + }); +} + +function normalizedString(value: unknown): string | null { + if (typeof value !== "string") return null; + const normalized = value.trim(); + return normalized.length > 0 ? normalized : null; +} + +export function resolveLaunchInteractiveCommand( + agent: AgentDefinition, + agentName: string, +): string | null { + return agentName === "nemocua" + ? agentRuntime.getTerminalCommand(agent, "interactive") + : agentRuntime.getInteractiveAgentCommand(agent, agentName); +} + +export function resolveTrustedLaunchAgent( + entry: SandboxEntry, + deps: LaunchReadinessHealthDeps, + agentName = normalizedString(entry.agent) ?? "openclaw", +): AgentDefinition { + const available = (deps.listAgents ?? listAgents)(); + if (!available.includes(agentName)) throw new LaunchReadinessObservationError("config"); + let agent: AgentDefinition; + try { + agent = (deps.loadAgent ?? loadAgent)(agentName); + } catch { + throw new LaunchReadinessEvidenceError(); + } + const interactive = resolveLaunchInteractiveCommand(agent, agentName); + if (!interactive) throw new LaunchReadinessObservationError("session"); + if (agentName === "nemocua" && interactive !== "nemocua interactive") { + throw new LaunchReadinessObservationError("session"); + } + return agent; +} + +function probeInferenceRoute( + sandboxName: string, + agent: InferenceRouteProbeAgent, + gatewayName: string, +): ReturnType { + return parseSandboxInferenceRouteProbeResult( + captureLaunchReadiness(buildSandboxInferenceRouteProbeArgs(sandboxName, agent, gatewayName), { + includeStreams: true, + }), + ); +} + +export async function requireLaunchSemanticHealth( + sandboxName: string, + gatewayName: string, + agentName: string, + entry: SandboxEntry, + agent: AgentDefinition, + inferenceConfigured: boolean, + deps: LaunchReadinessHealthDeps, +): Promise { + if (agentName === "nemocua") { + try { + (deps.cuaReadiness ?? requireCuaLifecycleReadiness)(entry); + } catch { + throw new LaunchReadinessObservationError("health"); + } + } else if (isTerminalAgent(agent)) { + const smoke = (deps.smoke ?? runAgentSmokeCommands)( + sandboxName, + agent, + (args, _options) => + (deps.capture ?? ((captureArgs) => captureLaunchReadiness(captureArgs)))(args), + gatewayName, + ); + if (!smoke.ok) { + const trustedExit = smoke.output?.match(/(?:^|\n)NEMOCLAW_AGENT_SMOKE_EXIT:(\d+)(?:\n|$)/); + if (!trustedExit) throw new LaunchReadinessEvidenceError(); + throw new LaunchReadinessObservationError("health"); + } + } else { + const running = await (deps.gatewayHealth ?? isSandboxGatewayRunningForStatus)( + sandboxName, + gatewayName, + ); + if (running === null) throw new LaunchReadinessEvidenceError(); + if (!running) throw new LaunchReadinessObservationError("health"); + const forwards = (deps.forwardsHealthy ?? areSandboxLaunchForwardsHealthy)( + sandboxName, + gatewayName, + ); + if (forwards === null) throw new LaunchReadinessEvidenceError(); + if (!forwards) { + throw new LaunchReadinessObservationError("health"); + } + } + if (inferenceConfigured) { + const inference = (deps.inferenceProbe ?? probeInferenceRoute)(sandboxName, agent, gatewayName); + const usable = inference.healthy && inference.httpStatus >= 200 && inference.httpStatus < 300; + if (usable) return; + if (inference.broken || (inference.httpStatus >= 100 && inference.httpStatus < 600)) { + throw new LaunchReadinessObservationError("health"); + } + throw new LaunchReadinessEvidenceError(); + } +} diff --git a/src/lib/actions/sandbox/launch.test.ts b/src/lib/actions/sandbox/launch.test.ts index 36f846d4964..c6a3c60c4a4 100644 --- a/src/lib/actions/sandbox/launch.test.ts +++ b/src/lib/actions/sandbox/launch.test.ts @@ -11,12 +11,19 @@ import type { SandboxEntry } from "../../state/registry"; const mocks = vi.hoisted(() => ({ calls: [] as string[], prepareInteractiveSession: vi.fn(), + printInteractiveSessionHints: vi.fn(), + completeInteractiveSessionSetup: vi.fn(), execSandbox: vi.fn(), prepareHermesLightTerminalSkin: vi.fn(), + inspectLaunchReadiness: vi.fn(), + publishLaunchReadiness: vi.fn(), + withLaunchReadinessMutationGate: vi.fn(), })); vi.mock("./connect", () => ({ prepareInteractiveSession: mocks.prepareInteractiveSession, + printInteractiveSessionHints: mocks.printInteractiveSessionHints, + completeInteractiveSessionSetup: mocks.completeInteractiveSessionSetup, })); vi.mock("./exec", () => ({ execSandbox: mocks.execSandbox, @@ -24,6 +31,17 @@ vi.mock("./exec", () => ({ vi.mock("./connect-hermes-light-skin", () => ({ prepareHermesLightTerminalSkin: mocks.prepareHermesLightTerminalSkin, })); +vi.mock("./launch-readiness", () => ({ + inspectLaunchReadiness: mocks.inspectLaunchReadiness, + publishLaunchReadiness: mocks.publishLaunchReadiness, + withLaunchReadinessMutationGate: mocks.withLaunchReadinessMutationGate, + publicationFromDecision: (sandboxName: string, decision: { fence?: { epochId: string } }) => ({ + sandboxName, + gatewayName: "nemoclaw", + gatewayPort: 8080, + epochId: decision.fence?.epochId ?? null, + }), +})); import { launchSandbox } from "./launch"; @@ -86,6 +104,20 @@ describe("launchSandbox", () => { mocks.prepareHermesLightTerminalSkin.mockImplementation(() => { mocks.calls.push("prepareHermesLightTerminalSkin"); }); + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "missing", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: false, + }); + mocks.publishLaunchReadiness.mockResolvedValue({ kind: "published" }); + mocks.withLaunchReadinessMutationGate.mockImplementation(async (_publication, operation) => ({ + kind: "entered", + value: await operation(), + })); // Production keeps OpenClaw null in getSessionAgent so its recovery path // continues to use the legacy defaults. The launch resolver must still // load OpenClaw's trusted manifest before choosing the interactive command. @@ -136,7 +168,13 @@ describe("launchSandbox", () => { headless_command: "nemocua headless", }, }; - prepareSession("nemocua", nemocua); + const cuaEntry = sandboxEntry("nemocua"); + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "accepted", + category: "accepted", + agent: nemocua, + sb: cuaEntry, + }); const events: string[] = []; const childStarted = deferred(); const releaseChild = deferred(); @@ -150,7 +188,7 @@ describe("launchSandbox", () => { }); const launch = launchSandbox("alpha", { - getSandbox: () => sandboxEntry("nemocua"), + getSandbox: () => cuaEntry, requireCuaReadiness, resolveSandboxGatewayName: () => "gateway-alpha", withGatewayRouteMutationLock, @@ -252,6 +290,230 @@ describe("launchSandbox", () => { "prepareHermesLightTerminalSkin", "execSandbox", ]); + expect(mocks.withLaunchReadinessMutationGate).toHaveBeenCalledWith( + expect.objectContaining({ epochId: null }), + expect.any(Function), + ); + }); + + it("rejects a sandbox missing from local state before readiness recovery (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "missing", + fence: null, + gatewayName: null, + gatewayPort: null, + fenceFailed: true, + recoveryBlocked: true, + }); + + await expect(launchSandbox("alpha;echo pwned")).rejects.toThrow( + "Sandbox 'alpha;echo pwned' is not registered in the local NemoClaw state.", + ); + + expect(mocks.withLaunchReadinessMutationGate).not.toHaveBeenCalled(); + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.execSandbox).not.toHaveBeenCalled(); + }); + + it("uses the accepted lease path without running the complete preflight (#8942)", async () => { + const openclaw = loadAgent("openclaw"); + const sb = sandboxEntry("openclaw"); + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "accepted", + category: "accepted", + agent: openclaw, + sb, + }); + + await launchSandbox("alpha"); + + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.printInteractiveSessionHints).toHaveBeenCalledWith("alpha"); + expect(mocks.completeInteractiveSessionSetup).toHaveBeenCalledWith("alpha", sb); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + expect(mocks.printInteractiveSessionHints).toHaveBeenCalledBefore( + mocks.completeInteractiveSessionSetup, + ); + expect(mocks.completeInteractiveSessionSetup).toHaveBeenCalledBefore( + mocks.prepareHermesLightTerminalSkin, + ); + expect(mocks.prepareHermesLightTerminalSkin).toHaveBeenCalledBefore(mocks.execSandbox); + expect(launchedCommand()).toEqual(["bash", "-lc", "openclaw tui"]); + }); + + it("does not mutate after its epoch is replaced by a newer accepted lease (#8942)", async () => { + const openclaw = loadAgent("openclaw"); + const sb = sandboxEntry("openclaw"); + mocks.inspectLaunchReadiness + .mockResolvedValueOnce({ + kind: "fallback", + category: "config", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }) + .mockResolvedValueOnce({ + kind: "accepted", + category: "accepted", + agent: openclaw, + sb, + }); + mocks.withLaunchReadinessMutationGate.mockResolvedValueOnce({ kind: "changed" }); + + await launchSandbox("alpha"); + + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + expect(mocks.inspectLaunchReadiness).toHaveBeenCalledTimes(2); + expect(mocks.withLaunchReadinessMutationGate).toHaveBeenCalledWith( + expect.objectContaining({ epochId: "a".repeat(64) }), + expect.any(Function), + ); + expect(mocks.completeInteractiveSessionSetup).toHaveBeenCalledWith("alpha", sb); + expect(mocks.execSandbox).toHaveBeenCalledOnce(); + }); + + it("keeps repeated launch and exit cycles on the accepted non-sliding lease (#8942)", async () => { + const openclaw = loadAgent("openclaw"); + const sb = sandboxEntry("openclaw"); + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "accepted", + category: "accepted", + agent: openclaw, + sb, + }); + + await launchSandbox("alpha"); + await launchSandbox("alpha"); + + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.execSandbox).toHaveBeenCalledTimes(2); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + }); + + it("publishes recaptured final state only after successful complete preflight (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "expired", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }); + + await launchSandbox("alpha"); + + expect(mocks.prepareInteractiveSession).toHaveBeenCalledBefore(mocks.publishLaunchReadiness); + expect(mocks.publishLaunchReadiness).toHaveBeenCalledBefore(mocks.execSandbox); + }); + + it("keeps ordinary launch available when evidence observation, hashing, or storage fails (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "unsafe", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }); + mocks.publishLaunchReadiness.mockResolvedValue({ kind: "evidence-failed" }); + + await expect(launchSandbox("alpha")).resolves.toBeUndefined(); + + expect(mocks.prepareInteractiveSession).toHaveBeenCalled(); + expect(mocks.execSandbox).toHaveBeenCalled(); + }); + + it("runs the complete preflight and interactive command when macOS evidence is unavailable (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: false, + authorityUnsupported: true, + }); + + await expect(launchSandbox("alpha")).resolves.toBeUndefined(); + + expect(mocks.prepareInteractiveSession).toHaveBeenCalledOnce(); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + expect(mocks.withLaunchReadinessMutationGate).toHaveBeenCalledWith( + expect.objectContaining({ epochId: null }), + expect.any(Function), + ); + expect(mocks.execSandbox).toHaveBeenCalledOnce(); + }); + + it("stops before the complete preflight when a prior launch-readiness epoch may remain acceptable (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "unsafe", + fence: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: true, + recoveryBlocked: true, + }); + + await expect(launchSandbox("alpha")).rejects.toThrow( + "Launch readiness evidence could not be safely invalidated", + ); + + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + expect(mocks.execSandbox).not.toHaveBeenCalled(); + }); + + it("stops before the complete preflight when the fenced epoch cannot be revalidated (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "config", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }); + mocks.withLaunchReadinessMutationGate.mockResolvedValue({ kind: "unsafe" }); + + await expect(launchSandbox("alpha")).rejects.toThrow( + "Launch readiness evidence could not be safely invalidated", + ); + + expect(mocks.prepareInteractiveSession).not.toHaveBeenCalled(); + expect(mocks.publishLaunchReadiness).not.toHaveBeenCalled(); + expect(mocks.execSandbox).not.toHaveBeenCalled(); + }); + + it("does not launch after final semantic validation reports unhealthy state (#8942)", async () => { + mocks.inspectLaunchReadiness.mockResolvedValue({ + kind: "fallback", + category: "health", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }); + mocks.publishLaunchReadiness.mockResolvedValue({ + kind: "validation-failed", + category: "health", + }); + + await expect(launchSandbox("alpha")).rejects.toThrow( + "Launch readiness final validation failed due to health", + ); + + expect(mocks.prepareInteractiveSession).toHaveBeenCalled(); + expect(mocks.execSandbox).not.toHaveBeenCalled(); }); it("does not print connect's in-sandbox command hint (#6006)", async () => { diff --git a/src/lib/actions/sandbox/launch.ts b/src/lib/actions/sandbox/launch.ts index 7275b7930e9..2bdb77c61d9 100644 --- a/src/lib/actions/sandbox/launch.ts +++ b/src/lib/actions/sandbox/launch.ts @@ -6,17 +6,31 @@ import { requireCuaLifecycleReadiness } from "../../cua/lifecycle-readiness"; import { resolveSandboxGatewayName } from "../../gateway-runtime-action"; import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; import { withMcpLifecycleLock as withSandboxMutationLock } from "../../state/mcp-lifecycle-lock-acquisition"; -import { prepareInteractiveSession } from "./connect"; +import { + completeInteractiveSessionSetup, + prepareInteractiveSession, + printInteractiveSessionHints, +} from "./connect"; import { prepareHermesLightTerminalSkin } from "./connect-hermes-light-skin"; import { execSandbox } from "./exec"; import { getKnownSandboxTarget } from "./gateway-target"; +import { + inspectLaunchReadiness, + publicationFromDecision, + publishLaunchReadiness, + withLaunchReadinessMutationGate, +} from "./launch-readiness"; + +const LAUNCH_READINESS_FENCE_REPAIR = + "Launch readiness evidence could not be safely invalidated. Repair the current user's secure OS runtime authority and NemoClaw state permissions, then retry."; /** * Connect to a sandbox and start its agent in one host-side step (#6006). * - * The preflight is the same one `connect` runs, and it must run first: the - * agent started over `exec` without process recovery renders a TUI that sits - * disconnected because the gateway was never checked or restarted. + * Launch either validates a launch-readiness lease or runs the same complete + * preflight as `connect` before starting the agent. Starting over `exec` + * without either path can leave the TUI disconnected from an unhealthy + * gateway. */ interface LaunchSandboxDeps { getSandbox?: typeof getKnownSandboxTarget; @@ -24,6 +38,9 @@ interface LaunchSandboxDeps { resolveSandboxGatewayName?: typeof resolveSandboxGatewayName; withGatewayRouteMutationLock?: typeof withGatewayRouteMutationLock; withSandboxMutationLock?: typeof withSandboxMutationLock; + inspectLaunchReadiness?: typeof inspectLaunchReadiness; + publishLaunchReadiness?: typeof publishLaunchReadiness; + withLaunchReadinessMutationGate?: typeof withLaunchReadinessMutationGate; } async function launchCuaUnderMutationLocks( @@ -58,7 +75,48 @@ export async function launchSandbox( sandboxName: string, deps: LaunchSandboxDeps = {}, ): Promise { - const { agent, sb } = await prepareInteractiveSession(sandboxName); + const inspect = deps.inspectLaunchReadiness ?? inspectLaunchReadiness; + const enterMutationGate = deps.withLaunchReadinessMutationGate ?? withLaunchReadinessMutationGate; + let decision = await inspect(sandboxName); + let session: Awaited>; + while (true) { + if (decision.kind === "accepted") { + printInteractiveSessionHints(sandboxName); + completeInteractiveSessionSetup(sandboxName, decision.sb); + session = { agent: decision.agent, sb: decision.sb }; + break; + } + if ( + decision.category === "missing" && + decision.gatewayName === null && + decision.gatewayPort === null + ) { + throw new Error(`Sandbox '${sandboxName}' is not registered in the local NemoClaw state.`); + } + if (decision.recoveryBlocked) throw new Error(LAUNCH_READINESS_FENCE_REPAIR); + const fallbackDecision = decision; + const publicationRequest = publicationFromDecision(sandboxName, fallbackDecision); + const gated = await enterMutationGate(publicationRequest, async () => { + const prepared = await prepareInteractiveSession(sandboxName); + const publication = fallbackDecision.fence + ? await (deps.publishLaunchReadiness ?? publishLaunchReadiness)(publicationRequest) + : null; + return { prepared, publication }; + }); + if (gated.kind === "changed") { + decision = await inspect(sandboxName); + continue; + } + if (gated.kind === "unsafe") throw new Error(LAUNCH_READINESS_FENCE_REPAIR); + if (gated.value.publication?.kind === "validation-failed") { + throw new Error( + `Launch readiness final validation failed due to ${gated.value.publication.category}. Retry launch.`, + ); + } + session = gated.value.prepared; + break; + } + const { agent, sb } = session; const isCua = sb?.agent === "nemocua"; const agentCommand = isCua ? agentRuntime.getTerminalCommand(agent, "interactive") diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 165e95ef3d4..f0b63008d45 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -271,10 +271,22 @@ export function executeGatewaySupervisorAction( async function executeSandboxExecCommandForStatus( sandboxName: string, command: string, + gatewayName?: string, + capture: typeof captureOpenshellForStatus = captureOpenshellForStatus, ): Promise { const markedCommand = buildSandboxExecMarkedCommand(command); - const result = await captureOpenshellForStatus( - ["sandbox", "exec", "--name", sandboxName, "--", "sh", "-c", markedCommand], + const result = await capture( + [ + "sandbox", + "exec", + "--name", + sandboxName, + ...(gatewayName ? ["-g", gatewayName] : []), + "--", + "sh", + "-c", + markedCommand, + ], { ignoreError: true }, ); if (isCommandTimeout(result) || result.error) return null; @@ -460,12 +472,20 @@ export function confirmRecoveredSandboxGatewayManaged( export async function isSandboxGatewayRunningForStatus( sandboxName: string, + gatewayName?: string, + options: { + getSessionAgent?: typeof agentRuntime.getSessionAgent; + capture?: typeof captureOpenshellForStatus; + getHealthProbeUrl?: typeof getSandboxHealthProbeUrl; + } = {}, ): Promise { - const agent = agentRuntime.getSessionAgent(sandboxName); + const agent = (options.getSessionAgent ?? agentRuntime.getSessionAgent)(sandboxName); if (agent && !agentRuntime.hasGatewayRuntime(agent)) return null; - const probeUrl = getSandboxHealthProbeUrl(sandboxName); + const probeUrl = (options.getHealthProbeUrl ?? getSandboxHealthProbeUrl)(sandboxName); const command = `HTTP_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 3 ${shellQuote(probeUrl)} 2>/dev/null || echo 000); case "$HTTP_CODE" in 200|401) echo RUNNING ;; *) echo STOPPED ;; esac`; - return parseSandboxGatewayProbe(await executeSandboxExecCommandForStatus(sandboxName, command)); + return parseSandboxGatewayProbe( + await executeSandboxExecCommandForStatus(sandboxName, command, gatewayName, options.capture), + ); } /** diff --git a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts index f876637a4f3..8fc2ea1e0ad 100644 --- a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts +++ b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts @@ -63,7 +63,10 @@ describe("sandbox recovery with a Hermes cron restore gate", () => { }); expect(events).toEqual(["prepare", "connect", "recover"]); expect(mocks.prepareHermesCronRestoreRecovery).toHaveBeenCalledWith("alpha"); - expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { probeOnly: true }); + expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { + probeOnly: true, + requireLaunchReadinessPublication: false, + }); expect(mocks.recoverHermesCronRestore).toHaveBeenCalledWith("alpha"); }); @@ -89,7 +92,10 @@ describe("sandbox recovery with a Hermes cron restore gate", () => { await recoverSandboxWithHermesCronRestore("alpha"); expect(mocks.prepareHermesCronRestoreRecovery).toHaveBeenCalledWith("alpha"); - expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { probeOnly: true }); + expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { + probeOnly: true, + requireLaunchReadinessPublication: false, + }); expect(mocks.recoverHermesCronRestore).toHaveBeenCalledWith("alpha"); }); @@ -98,7 +104,10 @@ describe("sandbox recovery with a Hermes cron restore gate", () => { await recoverSandboxWithHermesCronRestore("alpha"); - expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { probeOnly: true }); + expect(mocks.connectSandbox).toHaveBeenCalledWith("alpha", { + probeOnly: true, + requireLaunchReadinessPublication: false, + }); expect(mocks.prepareHermesCronRestoreRecovery).not.toHaveBeenCalled(); expect(mocks.recoverHermesCronRestore).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts index b318986ab5f..bd41aa21b0a 100644 --- a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts +++ b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts @@ -20,7 +20,10 @@ export async function recoverSandboxWithHermesCronRestore(sandboxName: string): if (agent?.name === "hermes") { prepareHermesCronRestoreRecovery(sandboxName); } - await connectSandbox(sandboxName, { probeOnly: true }); + await connectSandbox(sandboxName, { + probeOnly: true, + requireLaunchReadinessPublication: false, + }); if (agent?.name !== "hermes") return; const outcome = recoverHermesCronRestore(sandboxName); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index a1ad6c85382..d85930dd8bf 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -19,10 +19,11 @@ import { } from "./runtime/lifecycle-runtime"; function verifyGateway(sandboxName: string): Promise { - const { connectSandbox } = require("./connect") as { - connectSandbox: (name: string, options?: { probeOnly?: boolean }) => Promise; - }; - return connectSandbox(sandboxName, { probeOnly: true }); + const { connectSandbox } = require("./connect") as typeof import("./connect"); + return connectSandbox(sandboxName, { + probeOnly: true, + requireLaunchReadinessPublication: false, + }); } type SandboxStartupRecoveryResult = import("./connect").SandboxStartupRecoveryResult; diff --git a/src/lib/agent/terminal-smoke.test.ts b/src/lib/agent/terminal-smoke.test.ts index 1131b7e96dd..be2481983af 100644 --- a/src/lib/agent/terminal-smoke.test.ts +++ b/src/lib/agent/terminal-smoke.test.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { DCODE_MANAGED_EXEC_LAUNCHER } from "../actions/sandbox/connect-inference-route-probe"; -import type { AgentDefinition } from "./defs"; +import { type AgentDefinition, loadAgent } from "./defs"; import { buildAgentSmokeArgs, runAgentSmokeCommands } from "./terminal-smoke"; function agent(name: string): AgentDefinition { @@ -36,6 +36,35 @@ describe("terminal agent smoke command invocation", () => { expect(args.at(-1)).toBe("hermes --version"); }); + it("pins every smoke exec to the owning OpenShell gateway (#8942)", () => { + const capture = vi.fn((_args: string[]) => ({ + status: 0, + output: "NEMOCLAW_AGENT_SMOKE_BEGIN\nNEMOCLAW_AGENT_SMOKE_EXIT:0\n", + })); + + expect( + runAgentSmokeCommands( + "alpha", + loadAgent("langchain-deepagents-code"), + capture, + "nemoclaw-8091", + ), + ).toEqual({ ok: true }); + + expect(capture).toHaveBeenCalled(); + for (const [args] of capture.mock.calls) { + expect(args.slice(0, 7)).toEqual([ + "sandbox", + "exec", + "-n", + "alpha", + "-g", + "nemoclaw-8091", + "--no-tty", + ]); + } + }); + it("does not add a login shell to Deep Agents Code smoke exec (#8624)", () => { const issued: string[][] = []; const result = runAgentSmokeCommands( @@ -45,7 +74,7 @@ describe("terminal agent smoke command invocation", () => { issued.push(args); return { status: 0, - output: `NEMOCLAW_AGENT_SMOKE_BEGIN\nNEMOCLAW_AGENT_SMOKE_EXIT:0\n`, + output: "NEMOCLAW_AGENT_SMOKE_BEGIN\nNEMOCLAW_AGENT_SMOKE_EXIT:0\n", }; }, ); diff --git a/src/lib/agent/terminal-smoke.ts b/src/lib/agent/terminal-smoke.ts index 4b8e5af6ada..2a97f9af97c 100644 --- a/src/lib/agent/terminal-smoke.ts +++ b/src/lib/agent/terminal-smoke.ts @@ -58,6 +58,7 @@ export function buildAgentSmokeArgs( sandboxName: string, agent: AgentDefinition, command: string, + gatewayName?: string, ): string[] { if (agent.name === "langchain-deepagents-code") { return [ @@ -65,6 +66,7 @@ export function buildAgentSmokeArgs( "exec", "-n", sandboxName, + ...(gatewayName ? ["-g", gatewayName] : []), "--no-tty", "--env", "HOME=/usr/local/lib/nemoclaw", @@ -86,6 +88,7 @@ export function buildAgentSmokeArgs( "exec", "-n", sandboxName, + ...(gatewayName ? ["-g", gatewayName] : []), "--", "sh", "-lc", @@ -99,14 +102,18 @@ export function runAgentSmokeCommands( sandboxName: string, agent: AgentDefinition, runCaptureOpenshell: RunCaptureOpenshell, + gatewayName?: string, ): AgentSmokeCommandResult { // smoke_commands are shell-form commands from repository-shipped agents/*/manifest.yaml files. // Switch to argv-form commands before accepting custom or user-provided manifests here. const commands = agent.runtime?.smoke_commands ?? []; for (const command of commands) { - const result = runCaptureOpenshell(buildAgentSmokeArgs(sandboxName, agent, command), { - ignoreError: true, - }); + const result = runCaptureOpenshell( + buildAgentSmokeArgs(sandboxName, agent, command, gatewayName), + { + ignoreError: true, + }, + ); const output = typeof result === "string" ? result : (result?.output ?? null); const requireManagedBoundary = agent.name === "langchain-deepagents-code"; const exitCode = getSmokeExitCode(output, requireManagedBoundary); diff --git a/src/lib/state/launch-readiness-lease.test.ts b/src/lib/state/launch-readiness-lease.test.ts new file mode 100644 index 00000000000..a8ec06b8a0e --- /dev/null +++ b/src/lib/state/launch-readiness-lease.test.ts @@ -0,0 +1,776 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + checkLaunchReadinessMutationAuthority, + fenceLaunchReadinessLease as fenceLeaseStore, + LAUNCH_READINESS_LEASE_MS, + LAUNCH_READINESS_MAX_BYTES, + LaunchReadinessFenceError, + type LaunchReadinessIdentity, + type LaunchReadinessStoreOptions, + launchReadinessAuthorityPath, + launchReadinessReceiptPath, + publishLaunchReadinessLease as publishLeaseStore, + readLaunchReadinessLease as readLeaseStore, +} from "./launch-readiness-lease"; + +const SANDBOX = "alpha"; +const GATEWAY_PORT = 8080; +const GATEWAY_NAME = "nemoclaw"; +const EPOCH_A = "a".repeat(64); +const EPOCH_B = "b".repeat(64); +const EPOCH_C = "d".repeat(64); +const DIGEST = "c".repeat(64); + +function identity(gatewayName = GATEWAY_NAME): LaunchReadinessIdentity { + return { + registry: DIGEST, + agent: DIGEST, + livePolicy: DIGEST, + liveInference: DIGEST, + gatewayName, + lifecycleGeneration: "generation-1", + liveIdentityFingerprint: DIGEST, + }; +} + +function throwReadOnly(message: string): never { + throw Object.assign(new Error(message), { code: "EROFS" }); +} + +describe("launch readiness lease storage", () => { + let root: string; + let home: string; + let runtimeRoot: string; + let wallMs: number; + let uptimeMs: number; + let bootId: string; + let epochs: string[]; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launch-readiness-")); + home = path.join(root, "home"); + runtimeRoot = path.join(root, "runtime"); + fs.mkdirSync(home, { mode: 0o700 }); + fs.chmodSync(home, 0o700); + fs.mkdirSync(runtimeRoot, { mode: 0o700 }); + fs.chmodSync(runtimeRoot, 0o700); + wallMs = 2_000_000_000_000; + uptimeMs = 100_000; + bootId = "boot-a"; + epochs = [EPOCH_A, EPOCH_B]; + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + function options( + overrides: Partial = {}, + ): LaunchReadinessStoreOptions { + return { + home, + nowWallMs: () => wallMs, + nowUptimeMs: () => uptimeMs, + bootId: () => bootId, + uid: () => process.getuid?.() ?? 0, + randomEpoch: () => epochs.shift() ?? EPOCH_B, + runtimeAuthorityRoot: () => runtimeRoot, + ...overrides, + }; + } + + function fenceLaunchReadinessLease( + sandboxName: string, + gatewayPort: number, + storeOptions: LaunchReadinessStoreOptions, + ) { + return fenceLeaseStore(sandboxName, GATEWAY_NAME, gatewayPort, storeOptions); + } + + function readLaunchReadinessLease( + sandboxName: string, + gatewayPort: number, + storeOptions: LaunchReadinessStoreOptions, + ) { + return readLeaseStore(sandboxName, GATEWAY_NAME, gatewayPort, storeOptions); + } + + function publishLaunchReadinessLease( + sandboxName: string, + gatewayPort: number, + expectedEpochId: string, + launchIdentity: LaunchReadinessIdentity, + storeOptions: LaunchReadinessStoreOptions, + ) { + return publishLeaseStore( + sandboxName, + GATEWAY_NAME, + gatewayPort, + expectedEpochId, + launchIdentity, + storeOptions, + ); + } + + function publish(): ReturnType { + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + return publishLaunchReadinessLease(SANDBOX, GATEWAY_PORT, fence.epochId, identity(), options()); + } + + function restoreReceipt(targetHome: string, raw: string): void { + const target = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, targetHome); + fs.mkdirSync(path.dirname(target), { mode: 0o700, recursive: true }); + fs.chmodSync(path.dirname(target), 0o700); + fs.writeFileSync(target, raw, { mode: 0o600 }); + } + + function readInFreshProcess(gatewayName = GATEWAY_NAME, gatewayPort = GATEWAY_PORT): string { + const moduleUrl = pathToFileURL(path.resolve("src/lib/state/launch-readiness-lease.ts")).href; + const source = ` + import launchReadinessLease from ${JSON.stringify(moduleUrl)}; + const { readLaunchReadinessLease } = launchReadinessLease; + const result = readLaunchReadinessLease( + ${JSON.stringify(SANDBOX)}, + ${JSON.stringify(gatewayName)}, + ${gatewayPort}, + { + home: ${JSON.stringify(home)}, + nowWallMs: () => ${wallMs}, + nowUptimeMs: () => ${uptimeMs}, + bootId: () => ${JSON.stringify(bootId)}, + uid: () => ${process.getuid?.() ?? 0}, + runtimeAuthorityRoot: () => ${JSON.stringify(runtimeRoot)}, + }, + ); + process.stdout.write(result.kind); + `; + return execFileSync( + process.execPath, + ["--import", "tsx", "--input-type=module", "--eval", source], + { encoding: "utf8" }, + ); + } + + function expectFenceFailure(operation: () => unknown, blocksRecovery: boolean): void { + try { + operation(); + throw new Error("Expected launch-readiness fencing to fail."); + } catch (error) { + expect(error).toBeInstanceOf(LaunchReadinessFenceError); + expect((error as LaunchReadinessFenceError).blocksRecovery).toBe(blocksRecovery); + } + } + + it("publishes a fixed 24-hour lease and accepts it on the same boot and user", () => { + const lease = publish(); + expect(lease.leaseExpiresWallMs - lease.leaseStartedWallMs).toBe(LAUNCH_READINESS_LEASE_MS); + wallMs += 60_000; + uptimeMs += 60_000; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options())).toMatchObject({ + kind: "valid", + lease: { epochId: EPOCH_A, sandboxName: SANDBOX }, + }); + }); + + it.skipIf(process.platform !== "darwin")( + "disables macOS evidence despite caller-controlled runtime path variables (#8942)", + () => { + const sandboxName = `darwin-authority-${process.pid}-${Date.now()}`; + const productionOptions = options({ + runtimeAuthorityRoot: undefined, + randomEpoch: () => EPOCH_A, + }); + vi.stubEnv("TMPDIR", runtimeRoot); + vi.stubEnv("HOME", runtimeRoot); + vi.stubEnv("XDG_RUNTIME_DIR", runtimeRoot); + vi.stubEnv("DARWIN_USER_TEMP_DIR", runtimeRoot); + vi.stubEnv("NEMOCLAW_RUNTIME_AUTHORITY_ROOT", runtimeRoot); + vi.stubEnv("LAUNCHD_SOCKET", runtimeRoot); + vi.stubEnv("SECURITYSESSIONID", "caller-session"); + vi.stubEnv("__CF_USER_TEXT_ENCODING", "caller-encoding"); + + expectFenceFailure( + () => fenceLeaseStore(sandboxName, GATEWAY_NAME, GATEWAY_PORT, productionOptions), + false, + ); + + const testAuthorityOptions = options({ randomEpoch: () => EPOCH_A }); + const fence = fenceLeaseStore(sandboxName, GATEWAY_NAME, GATEWAY_PORT, testAuthorityOptions); + publishLeaseStore( + sandboxName, + GATEWAY_NAME, + GATEWAY_PORT, + fence.epochId, + identity(), + testAuthorityOptions, + ); + expect(readLeaseStore(sandboxName, GATEWAY_NAME, GATEWAY_PORT, productionOptions).kind).toBe( + "unsafe", + ); + expect( + checkLaunchReadinessMutationAuthority( + sandboxName, + GATEWAY_NAME, + GATEWAY_PORT, + null, + productionOptions, + ), + ).toBe("current"); + }, + ); + + it("preserves the original lease envelope when the complete preflight republishes before expiry", () => { + const first = publish(); + wallMs += 60 * 60_000; + uptimeMs += 60 * 60_000; + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + wallMs += 60 * 60_000; + uptimeMs += 60 * 60_000; + const second = publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + fence.epochId, + identity(), + options(), + ); + expect(second.epochId).toBe(EPOCH_B); + expect(second.leaseStartedWallMs).toBe(first.leaseStartedWallMs); + expect(second.leaseExpiresWallMs).toBe(first.leaseExpiresWallMs); + }); + + it("starts a new envelope only after the prior lease expires", () => { + const first = publish(); + wallMs = first.leaseExpiresWallMs; + uptimeMs += LAUNCH_READINESS_LEASE_MS; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("expired"); + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + const second = publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + fence.epochId, + identity(), + options(), + ); + expect(second.leaseStartedWallMs).toBe(wallMs); + expect(second.leaseExpiresWallMs).toBe(wallMs + LAUNCH_READINESS_LEASE_MS); + }); + + it("rejects rollback, future publication, and the stricter monotonic expiry", () => { + const lease = publish(); + wallMs = lease.publishedWallMs - 1; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("malformed"); + wallMs = lease.publishedWallMs + 1; + uptimeMs = lease.publishedUptimeMs - 1; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("malformed"); + uptimeMs = lease.publishedUptimeMs + LAUNCH_READINESS_LEASE_MS + 1; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("expired"); + }); + + it("rejects non-finite, negative, and inconsistent time records", () => { + expect(() => + fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options({ nowWallMs: () => Number.NaN })), + ).toThrow(); + expect(() => + fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options({ nowUptimeMs: () => -1 })), + ).toThrow(); + + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const value = JSON.parse(fs.readFileSync(receiptPath, "utf8")) as Record; + value.leaseExpiresWallMs = Number(value.leaseExpiresWallMs) + 1; + fs.writeFileSync(receiptPath, JSON.stringify(value), { mode: 0o600 }); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("malformed"); + }); + + it("durably fences clock rollback without starting a replacement envelope", () => { + const original = publish(); + wallMs = original.publishedWallMs - 1; + const rollbackFence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(rollbackFence).toMatchObject({ + publicationState: "time-unsafe", + preservedLeaseStartedWallMs: original.leaseStartedWallMs, + preservedLeaseExpiresWallMs: original.leaseExpiresWallMs, + preservedLeaseElapsedMs: 0, + }); + expect(() => + publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + rollbackFence.epochId, + identity(), + options(), + ), + ).toThrow("disabled while authority or clock history is unsafe"); + + wallMs = original.publishedWallMs + 1; + uptimeMs += 2; + const repeatedFence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(repeatedFence.publicationState).toBe("time-unsafe"); + expect(() => + publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + repeatedFence.epochId, + identity(), + options(), + ), + ).toThrow("disabled while authority or clock history is unsafe"); + + wallMs = original.leaseExpiresWallMs; + uptimeMs += LAUNCH_READINESS_LEASE_MS; + const expiredFence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(expiredFence).toMatchObject({ + publicationState: "ready", + preservedLeaseStartedWallMs: null, + preservedLeaseExpiresWallMs: null, + preservedLeaseElapsedMs: null, + }); + }); + + it("carries the stricter monotonic elapsed duration across republication", () => { + publish(); + wallMs += 60 * 60_000; + uptimeMs += 2 * 60 * 60_000; + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(fence.preservedLeaseElapsedMs).toBe(2 * 60 * 60_000); + + const second = publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + fence.epochId, + identity(), + options(), + ); + expect(second.elapsedAtPublicationMs).toBe(2 * 60 * 60_000); + + wallMs += 21 * 60 * 60_000; + uptimeMs += 22 * 60 * 60_000; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("expired"); + }); + + it("rejects reboot and restored state or home volumes", () => { + publish(); + bootId = "boot-b"; + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("identity"); + + bootId = "boot-a"; + const stateRoot = path.join(home, ".nemoclaw"); + const savedState = path.join(root, "saved-state"); + const raw = fs.readFileSync(launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home), "utf8"); + fs.renameSync(stateRoot, savedState); + restoreReceipt(home, raw); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("identity"); + + const replacementHome = path.join(root, "replacement-home"); + fs.mkdirSync(replacementHome, { mode: 0o700 }); + restoreReceipt(replacementHome, raw); + expect( + readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options({ home: replacementHome })).kind, + ).toBe("identity"); + }); + + it("uses the random fence epoch as publication CAS authority", () => { + const stale = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + const current = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(() => + publishLaunchReadinessLease(SANDBOX, GATEWAY_PORT, stale.epochId, identity(), options()), + ).toThrow("authority changed"); + expect( + publishLaunchReadinessLease(SANDBOX, GATEWAY_PORT, current.epochId, identity(), options()) + .epochId, + ).toBe(current.epochId); + }); + + it("keeps the newer published lease valid when a paused producer resumes stale", () => { + const pausedProducer = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + const currentProducer = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + currentProducer.epochId, + identity(), + options(), + ); + + expect( + checkLaunchReadinessMutationAuthority( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + pausedProducer.epochId, + options(), + ), + ).toBe("changed"); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options())).toMatchObject({ + kind: "valid", + lease: { epochId: currentProducer.epochId }, + }); + expect(() => + publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + pausedProducer.epochId, + identity(), + options(), + ), + ).toThrow("authority changed"); + }); + + it("revalidates the runtime epoch used to enter the recovery mutation window", () => { + const stale = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect( + checkLaunchReadinessMutationAuthority( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + stale.epochId, + options(), + ), + ).toBe("current"); + + const current = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect( + checkLaunchReadinessMutationAuthority( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + stale.epochId, + options(), + ), + ).toBe("changed"); + expect( + checkLaunchReadinessMutationAuthority( + SANDBOX, + GATEWAY_NAME, + GATEWAY_PORT, + current.epochId, + options(), + ), + ).toBe("current"); + }); + + it("revalidates authoritative absence under the recovery mutation gate", () => { + expect( + checkLaunchReadinessMutationAuthority(SANDBOX, GATEWAY_NAME, GATEWAY_PORT, null, options()), + ).toBe("current"); + + fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect( + checkLaunchReadinessMutationAuthority(SANDBOX, GATEWAY_NAME, GATEWAY_PORT, null, options()), + ).toBe("changed"); + }); + + it("rejects a copied fence after the state volume changes during preflight", () => { + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + const stateRoot = path.join(home, ".nemoclaw"); + const savedState = path.join(root, "preflight-state"); + const raw = fs.readFileSync(launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home), "utf8"); + fs.renameSync(stateRoot, savedState); + restoreReceipt(home, raw); + + expect(() => + publishLaunchReadinessLease(SANDBOX, GATEWAY_PORT, fence.epochId, identity(), options()), + ).toThrow("authority changed"); + }); + + it("rejects unknown schema fields and reads only the bounded exact file", () => { + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const value = JSON.parse(fs.readFileSync(receiptPath, "utf8")) as Record; + value.extra = true; + fs.writeFileSync(receiptPath, JSON.stringify(value), { mode: 0o600 }); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("malformed"); + + fs.writeFileSync(receiptPath, "x".repeat(LAUNCH_READINESS_MAX_BYTES + 1), { mode: 0o600 }); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + }); + + it("requires an exact bounded private runtime-authority record", () => { + publish(); + const authorityPath = launchReadinessAuthorityPath(SANDBOX, runtimeRoot); + expect(path.basename(authorityPath)).toMatch(/^[a-f0-9]{64}\.json$/); + expect(fs.statSync(path.dirname(authorityPath)).mode & 0o777).toBe(0o700); + const authorityFd = fs.openSync(authorityPath, fs.constants.O_RDWR | fs.constants.O_NOFOLLOW); + try { + expect(fs.fstatSync(authorityFd).mode & 0o777).toBe(0o600); + + const authority = JSON.parse(fs.readFileSync(authorityFd, "utf8")) as Record; + authority.extra = true; + fs.ftruncateSync(authorityFd, 0); + fs.writeSync(authorityFd, JSON.stringify(authority), 0, "utf8"); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + + fs.ftruncateSync(authorityFd, 0); + fs.writeSync(authorityFd, "x".repeat(LAUNCH_READINESS_MAX_BYTES + 1), 0, "utf8"); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + } finally { + fs.closeSync(authorityFd); + } + }); + + it("rejects an authority copied to a restored runtime root", () => { + publish(); + const authorityPath = launchReadinessAuthorityPath(SANDBOX, runtimeRoot); + const authority = fs.readFileSync(authorityPath, "utf8"); + const replacementRuntime = path.join(root, "replacement-runtime"); + const replacementPath = launchReadinessAuthorityPath(SANDBOX, replacementRuntime); + fs.mkdirSync(path.dirname(replacementPath), { mode: 0o700, recursive: true }); + fs.chmodSync(replacementRuntime, 0o700); + fs.chmodSync(path.join(replacementRuntime, "nemoclaw"), 0o700); + fs.chmodSync(path.dirname(replacementPath), 0o700); + fs.writeFileSync(replacementPath, authority, { mode: 0o600 }); + + expect( + readLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + options({ runtimeAuthorityRoot: () => replacementRuntime }), + ).kind, + ).toBe("identity"); + }); + + it("quarantines unsafe authority history for one fixed non-sliding 24-hour interval before a new envelope", () => { + publish(); + const authorityPath = launchReadinessAuthorityPath(SANDBOX, runtimeRoot); + fs.writeFileSync(authorityPath, "{}\n", { mode: 0o600 }); + epochs = [EPOCH_B, EPOCH_C, "e".repeat(64)]; + + const quarantined = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(quarantined).toMatchObject({ + publicationState: "time-unsafe", + preservedLeaseStartedWallMs: wallMs, + preservedLeaseExpiresWallMs: wallMs + LAUNCH_READINESS_LEASE_MS, + preservedLeaseElapsedMs: 0, + }); + expect(() => + publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + quarantined.epochId, + identity(), + options(), + ), + ).toThrow("disabled while authority or clock history is unsafe"); + + wallMs += LAUNCH_READINESS_LEASE_MS; + uptimeMs += LAUNCH_READINESS_LEASE_MS - 1; + const stillQuarantined = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(stillQuarantined).toMatchObject({ + publicationState: "time-unsafe", + preservedLeaseStartedWallMs: quarantined.preservedLeaseStartedWallMs, + preservedLeaseExpiresWallMs: quarantined.preservedLeaseExpiresWallMs, + preservedLeaseElapsedMs: LAUNCH_READINESS_LEASE_MS - 1, + }); + + uptimeMs += 1; + const recovered = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + expect(recovered).toMatchObject({ + publicationState: "ready", + preservedLeaseStartedWallMs: null, + preservedLeaseExpiresWallMs: null, + preservedLeaseElapsedMs: null, + }); + const lease = publishLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + recovered.epochId, + identity(), + options(), + ); + expect(lease.leaseStartedWallMs).toBe(wallMs); + expect(lease.leaseExpiresWallMs).toBe(wallMs + LAUNCH_READINESS_LEASE_MS); + }); + + it("rejects unsafe receipt permissions and foreign ownership authority", () => { + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const receiptDir = path.dirname(receiptPath); + expect(fs.statSync(receiptDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(receiptPath).mode & 0o777).toBe(0o600); + + fs.chmodSync(receiptPath, 0o640); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + fs.chmodSync(receiptPath, 0o600); + + fs.chmodSync(receiptDir, 0o750); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + fs.chmodSync(receiptDir, 0o700); + + const stateAncestor = path.dirname(path.dirname(receiptDir)); + fs.chmodSync(stateAncestor, 0o770); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + fs.chmodSync(stateAncestor, 0o700); + + expect( + readLaunchReadinessLease( + SANDBOX, + GATEWAY_PORT, + options({ uid: () => (process.getuid?.() ?? 0) + 1 }), + ).kind, + ).toBe("unsafe"); + }); + + it("rejects symlinked ancestors, symlinked receipts, and hard links", () => { + const linkHome = path.join(root, "link-home"); + fs.mkdirSync(linkHome, { mode: 0o700 }); + const target = path.join(root, "target-state"); + fs.mkdirSync(target, { mode: 0o700 }); + fs.symlinkSync(target, path.join(linkHome, ".nemoclaw")); + expect(() => + fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options({ home: linkHome })), + ).toThrow(); + + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const saved = `${receiptPath}.saved`; + fs.renameSync(receiptPath, saved); + fs.symlinkSync(saved, receiptPath); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + fs.unlinkSync(receiptPath); + fs.renameSync(saved, receiptPath); + fs.linkSync(receiptPath, `${receiptPath}.link`); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("unsafe"); + }); + + it("uses one sandbox-global authority epoch across owning gateway changes", () => { + const gatewayA = "gateway-a"; + const gatewayB = "gateway-b"; + const portA = 8080; + const portB = 8081; + const fenceA = fenceLeaseStore(SANDBOX, gatewayA, portA, options()); + const first = publishLeaseStore( + SANDBOX, + gatewayA, + portA, + fenceA.epochId, + identity(gatewayA), + options(), + ); + wallMs += 60_000; + uptimeMs += 60_000; + + const fenceB = fenceLeaseStore(SANDBOX, gatewayB, portB, options()); + expect(fenceB.epochId).toBe(EPOCH_B); + expect(fenceB.preservedLeaseStartedWallMs).toBe(first.leaseStartedWallMs); + expect(fenceB.preservedLeaseExpiresWallMs).toBe(first.leaseExpiresWallMs); + expect(readLeaseStore(SANDBOX, gatewayA, portA, options()).kind).toBe("identity"); + expect(readInFreshProcess(gatewayA, portA)).toBe("identity"); + }); + + it("blocks when a prior runtime epoch exists but its rotation cannot be made durable", () => { + const fence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + fs.unlinkSync(launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home)); + const authorityPath = launchReadinessAuthorityPath(SANDBOX, runtimeRoot); + const rename = fs.renameSync.bind(fs); + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => + destination === authorityPath + ? throwReadOnly("read-only runtime authority") + : rename(source, destination), + ); + + expect(fence.epochId).toBe(EPOCH_A); + expectFenceFailure(() => fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()), true); + }); + + it("allows the complete preflight only when runtime authority and persistent evidence are securely absent", () => { + const mkdir = fs.mkdirSync.bind(fs); + vi.spyOn(fs, "mkdirSync").mockImplementation((target, mkdirOptions) => + String(target).startsWith(runtimeRoot) + ? throwReadOnly("read-only runtime root") + : mkdir(target, mkdirOptions as fs.MakeDirectoryOptions & { recursive: true }), + ); + + expectFenceFailure(() => fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()), false); + expect(fs.existsSync(launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home))).toBe(false); + expect(fs.existsSync(launchReadinessAuthorityPath(SANDBOX, runtimeRoot))).toBe(false); + }); + + it("blocks when runtime authority is unsafe even though persistent evidence is missing", () => { + fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + fs.unlinkSync(launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home)); + const authorityDir = path.dirname(launchReadinessAuthorityPath(SANDBOX, runtimeRoot)); + fs.chmodSync(authorityDir, 0o500); + + expectFenceFailure(() => fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()), true); + fs.chmodSync(authorityDir, 0o700); + }); + + it.each(["file", "directory", "ancestor"] as const)( + "keeps a restored unsafe persistent %s invalid across processes", + (unsafePart) => { + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const receiptDir = path.dirname(receiptPath); + const target = + unsafePart === "file" + ? receiptPath + : unsafePart === "directory" + ? receiptDir + : path.dirname(receiptDir); + const before = fs.statSync(receiptPath); + const originalMode = fs.statSync(target).mode & 0o777; + fs.chmodSync(target, unsafePart === "file" ? 0o640 : 0o770); + + expectFenceFailure(() => fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()), false); + fs.chmodSync(target, originalMode); + const restored = fs.statSync(receiptPath); + expect({ dev: restored.dev, ino: restored.ino }).toEqual({ + dev: before.dev, + ino: before.ino, + }); + expect(readInFreshProcess()).toBe("identity"); + + epochs = [EPOCH_C]; + const newFence = fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()); + publishLaunchReadinessLease(SANDBOX, GATEWAY_PORT, newFence.epochId, identity(), options()); + expect(readInFreshProcess()).toBe("valid"); + }, + ); + + it("rejects the unchanged persistent inode after simulated read-only remount fencing", () => { + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + const before = fs.statSync(receiptPath); + const rename = fs.renameSync.bind(fs); + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => + destination === receiptPath + ? throwReadOnly("read-only persistent state") + : rename(source, destination), + ); + + expectFenceFailure(() => fenceLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()), false); + vi.restoreAllMocks(); + const restored = fs.statSync(receiptPath); + expect({ dev: restored.dev, ino: restored.ino, ctimeMs: restored.ctimeMs }).toEqual({ + dev: before.dev, + ino: before.ino, + ctimeMs: before.ctimeMs, + }); + expect(readInFreshProcess()).toBe("identity"); + }); + + it("stores receipts by a SHA-256 key while verifying the exact sandbox name", () => { + publish(); + const receiptPath = launchReadinessReceiptPath(SANDBOX, GATEWAY_PORT, home); + expect(path.basename(receiptPath)).toMatch(/^[a-f0-9]{64}\.json$/); + expect(readLaunchReadinessLease("beta", GATEWAY_PORT, options()).kind).toBe("missing"); + + const value = JSON.parse(fs.readFileSync(receiptPath, "utf8")) as Record; + value.sandboxName = "beta"; + fs.writeFileSync(receiptPath, JSON.stringify(value), { mode: 0o600 }); + fs.chmodSync(receiptPath, 0o600); + expect(readLaunchReadinessLease(SANDBOX, GATEWAY_PORT, options()).kind).toBe("missing"); + }); +}); diff --git a/src/lib/state/launch-readiness-lease.ts b/src/lib/state/launch-readiness-lease.ts new file mode 100644 index 00000000000..6ce06422036 --- /dev/null +++ b/src/lib/state/launch-readiness-lease.ts @@ -0,0 +1,1591 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { nemoclawStateRoot } from "./state-root"; + +export const LAUNCH_READINESS_LEASE_MS = 24 * 60 * 60 * 1_000; +export const LAUNCH_READINESS_SCHEMA_VERSION = 1; +export const LAUNCH_READINESS_MAX_BYTES = 16 * 1_024; + +const RECEIPT_DIRECTORY = "launch-readiness"; +const AUTHORITY_PRODUCT_DIRECTORY = "nemoclaw"; +const AUTHORITY_DIRECTORY = "launch-readiness"; +const SHA256_RE = /^[a-f0-9]{64}$/; +const BOOT_ID_RE = /^[A-Za-z0-9._:-]{1,160}$/; + +export interface LaunchReadinessIdentity { + registry: string; + agent: string; + livePolicy: string; + liveInference: string; + gatewayName: string; + lifecycleGeneration: string; + liveIdentityFingerprint: string; +} + +export interface LaunchReadinessLease { + schemaVersion: 1; + kind: "lease"; + epochId: string; + sandboxName: string; + leaseStartedWallMs: number; + leaseExpiresWallMs: number; + elapsedAtPublicationMs: number; + publishedWallMs: number; + publishedUptimeMs: number; + bootId: string; + uid: number; + homeDevice: string; + homeInode: string; + storeDevice: string; + storeInode: string; + gatewayName: string; + gatewayPort: number; + identity: LaunchReadinessIdentity; +} + +export interface LaunchReadinessFence { + schemaVersion: 1; + kind: "fence"; + epochId: string; + sandboxName: string; + fencedWallMs: number; + fencedUptimeMs: number; + bootId: string; + uid: number; + homeDevice: string; + homeInode: string; + storeDevice: string; + storeInode: string; + gatewayName: string; + gatewayPort: number; + publicationState: "ready" | "time-unsafe"; + preservedLeaseStartedWallMs: number | null; + preservedLeaseExpiresWallMs: number | null; + preservedLeaseElapsedMs: number | null; +} + +export type LaunchReadinessRecord = LaunchReadinessLease | LaunchReadinessFence; + +export type LaunchReadinessLeaseRead = + | { kind: "missing" } + | { kind: "unsafe" } + | { kind: "malformed" } + | { kind: "expired"; lease: LaunchReadinessLease } + | { kind: "identity"; lease: LaunchReadinessLease } + | { kind: "valid"; lease: LaunchReadinessLease }; + +export type LaunchReadinessMutationAuthorityCheck = "current" | "changed" | "unsafe"; + +export interface LaunchReadinessStoreOptions { + home?: string; + nowWallMs?: () => number; + nowUptimeMs?: () => number; + bootId?: () => string | null; + uid?: () => number | null; + randomEpoch?: () => string; + /** Test seam only. Production derives the runtime authority root from the OS. */ + runtimeAuthorityRoot?: () => string | null; +} + +interface LaunchReadinessAuthority { + schemaVersion: 1; + kind: "authority"; + phase: "fence" | "lease"; + epochId: string; + sandboxName: string; + gatewayName: string; + gatewayPort: number; + observedWallMs: number; + observedUptimeMs: number; + bootId: string; + uid: number; + runtimeRootDevice: string; + runtimeRootInode: string; + storeDevice: string; + storeInode: string; + publicationState: "ready" | "time-unsafe"; + preservedLeaseStartedWallMs: number | null; + preservedLeaseExpiresWallMs: number | null; + preservedLeaseElapsedMs: number | null; +} + +interface BaseContext { + uid: number; + nowWallMs: number; + nowUptimeMs: number; + bootId: string; + randomEpoch: () => string; + runtimeAuthorityRoot: (() => string | null) | null; +} + +interface StoreContext extends BaseContext { + home: string; + stateRoot: string; + receiptDir: string; + receiptPath: string; + homeDevice: string; + homeInode: string; +} + +interface AuthorityContext extends BaseContext { + runtimeRoot: string; + authorityDir: string; + authorityPath: string; + runtimeRootDevice: string; + runtimeRootInode: string; +} + +interface SecureDirectory { + fd: number; + stat: fs.Stats; + path: string; + ancestors: Array<{ path: string; stat: fs.Stats; privateDirectory: boolean }>; +} + +class MissingStoreError extends Error {} +class UnsupportedAuthorityError extends Error {} +class UnsafeReceiptError extends Error {} +class MalformedReceiptError extends Error {} + +export class LaunchReadinessFenceError extends Error { + constructor( + readonly blocksRecovery: boolean, + readonly priorEpochInvalidated: boolean, + readonly authorityUnsupported = false, + ) { + super( + blocksRecovery + ? "Launch readiness evidence cannot be safely invalidated." + : "Launch readiness evidence storage is unavailable.", + ); + this.name = "LaunchReadinessFenceError"; + } +} + +function isPlainRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function hasExactKeys(value: Record, keys: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + return actual.length === expected.length && actual.every((key, index) => key === expected[index]); +} + +function isSafeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function isEpoch(value: unknown): value is string { + return typeof value === "string" && SHA256_RE.test(value); +} + +function isIdentity(value: unknown): value is LaunchReadinessIdentity { + if (!isPlainRecord(value)) return false; + if ( + !hasExactKeys(value, [ + "registry", + "agent", + "livePolicy", + "liveInference", + "gatewayName", + "lifecycleGeneration", + "liveIdentityFingerprint", + ]) + ) { + return false; + } + return ( + typeof value.gatewayName === "string" && + value.gatewayName.length > 0 && + value.gatewayName.length <= 256 && + typeof value.lifecycleGeneration === "string" && + value.lifecycleGeneration.length > 0 && + value.lifecycleGeneration.length <= 256 && + typeof value.liveIdentityFingerprint === "string" && + SHA256_RE.test(value.liveIdentityFingerprint) && + typeof value.registry === "string" && + SHA256_RE.test(value.registry) && + typeof value.agent === "string" && + SHA256_RE.test(value.agent) && + typeof value.livePolicy === "string" && + SHA256_RE.test(value.livePolicy) && + typeof value.liveInference === "string" && + SHA256_RE.test(value.liveInference) + ); +} + +function parseRecord(raw: string): LaunchReadinessRecord { + let value: unknown; + try { + value = JSON.parse(raw) as unknown; + } catch { + throw new MalformedReceiptError(); + } + if (!isPlainRecord(value) || value.schemaVersion !== LAUNCH_READINESS_SCHEMA_VERSION) { + throw new MalformedReceiptError(); + } + if (value.kind === "lease") { + if ( + !hasExactKeys(value, [ + "schemaVersion", + "kind", + "epochId", + "sandboxName", + "leaseStartedWallMs", + "leaseExpiresWallMs", + "elapsedAtPublicationMs", + "publishedWallMs", + "publishedUptimeMs", + "bootId", + "uid", + "homeDevice", + "homeInode", + "storeDevice", + "storeInode", + "gatewayName", + "gatewayPort", + "identity", + ]) || + !isEpoch(value.epochId) || + typeof value.sandboxName !== "string" || + value.sandboxName.length === 0 || + value.sandboxName.length > 256 || + !isSafeInteger(value.leaseStartedWallMs) || + !isSafeInteger(value.leaseExpiresWallMs) || + !isSafeInteger(value.elapsedAtPublicationMs) || + !isSafeInteger(value.publishedWallMs) || + !isSafeInteger(value.publishedUptimeMs) || + typeof value.bootId !== "string" || + !BOOT_ID_RE.test(value.bootId) || + !isSafeInteger(value.uid) || + typeof value.homeDevice !== "string" || + !/^\d+$/.test(value.homeDevice) || + typeof value.homeInode !== "string" || + !/^\d+$/.test(value.homeInode) || + typeof value.storeDevice !== "string" || + !/^\d+$/.test(value.storeDevice) || + typeof value.storeInode !== "string" || + !/^\d+$/.test(value.storeInode) || + typeof value.gatewayName !== "string" || + value.gatewayName.length === 0 || + value.gatewayName.length > 256 || + !Number.isInteger(value.gatewayPort) || + (value.gatewayPort as number) < 1 || + (value.gatewayPort as number) > 65535 || + !isIdentity(value.identity) + ) { + throw new MalformedReceiptError(); + } + return value as unknown as LaunchReadinessLease; + } + if (value.kind === "fence") { + if ( + !hasExactKeys(value, [ + "schemaVersion", + "kind", + "epochId", + "sandboxName", + "fencedWallMs", + "fencedUptimeMs", + "bootId", + "uid", + "homeDevice", + "homeInode", + "storeDevice", + "storeInode", + "gatewayName", + "gatewayPort", + "publicationState", + "preservedLeaseStartedWallMs", + "preservedLeaseExpiresWallMs", + "preservedLeaseElapsedMs", + ]) || + !isEpoch(value.epochId) || + typeof value.sandboxName !== "string" || + value.sandboxName.length === 0 || + value.sandboxName.length > 256 || + !isSafeInteger(value.fencedWallMs) || + !isSafeInteger(value.fencedUptimeMs) || + typeof value.bootId !== "string" || + !BOOT_ID_RE.test(value.bootId) || + !isSafeInteger(value.uid) || + typeof value.homeDevice !== "string" || + !/^\d+$/.test(value.homeDevice) || + typeof value.homeInode !== "string" || + !/^\d+$/.test(value.homeInode) || + typeof value.storeDevice !== "string" || + !/^\d+$/.test(value.storeDevice) || + typeof value.storeInode !== "string" || + !/^\d+$/.test(value.storeInode) || + typeof value.gatewayName !== "string" || + value.gatewayName.length === 0 || + value.gatewayName.length > 256 || + !Number.isInteger(value.gatewayPort) || + (value.gatewayPort as number) < 1 || + (value.gatewayPort as number) > 65535 || + (value.publicationState !== "ready" && value.publicationState !== "time-unsafe") || + !( + value.preservedLeaseStartedWallMs === null || + isSafeInteger(value.preservedLeaseStartedWallMs) + ) || + !( + value.preservedLeaseExpiresWallMs === null || + isSafeInteger(value.preservedLeaseExpiresWallMs) + ) || + !(value.preservedLeaseElapsedMs === null || isSafeInteger(value.preservedLeaseElapsedMs)) + ) { + throw new MalformedReceiptError(); + } + const hasStart = value.preservedLeaseStartedWallMs !== null; + const hasExpiry = value.preservedLeaseExpiresWallMs !== null; + const hasElapsed = value.preservedLeaseElapsedMs !== null; + if (hasStart !== hasExpiry || hasStart !== hasElapsed) throw new MalformedReceiptError(); + if ( + hasStart && + (value.preservedLeaseExpiresWallMs as number) - + (value.preservedLeaseStartedWallMs as number) !== + LAUNCH_READINESS_LEASE_MS + ) { + throw new MalformedReceiptError(); + } + if (hasElapsed && (value.preservedLeaseElapsedMs as number) > LAUNCH_READINESS_LEASE_MS) { + throw new MalformedReceiptError(); + } + return value as unknown as LaunchReadinessFence; + } + throw new MalformedReceiptError(); +} + +function parseAuthority(raw: string): LaunchReadinessAuthority { + let value: unknown; + try { + value = JSON.parse(raw) as unknown; + } catch { + throw new MalformedReceiptError(); + } + if ( + !isPlainRecord(value) || + !hasExactKeys(value, [ + "schemaVersion", + "kind", + "phase", + "epochId", + "sandboxName", + "gatewayName", + "gatewayPort", + "observedWallMs", + "observedUptimeMs", + "bootId", + "uid", + "runtimeRootDevice", + "runtimeRootInode", + "storeDevice", + "storeInode", + "publicationState", + "preservedLeaseStartedWallMs", + "preservedLeaseExpiresWallMs", + "preservedLeaseElapsedMs", + ]) || + value.schemaVersion !== 1 || + value.kind !== "authority" || + (value.phase !== "fence" && value.phase !== "lease") || + !isEpoch(value.epochId) || + typeof value.sandboxName !== "string" || + value.sandboxName.length === 0 || + value.sandboxName.length > 256 || + typeof value.gatewayName !== "string" || + value.gatewayName.length === 0 || + value.gatewayName.length > 256 || + !Number.isInteger(value.gatewayPort) || + (value.gatewayPort as number) < 1 || + (value.gatewayPort as number) > 65535 || + !isSafeInteger(value.observedWallMs) || + !isSafeInteger(value.observedUptimeMs) || + typeof value.bootId !== "string" || + !BOOT_ID_RE.test(value.bootId) || + !isSafeInteger(value.uid) || + typeof value.runtimeRootDevice !== "string" || + !/^\d+$/.test(value.runtimeRootDevice) || + typeof value.runtimeRootInode !== "string" || + !/^\d+$/.test(value.runtimeRootInode) || + typeof value.storeDevice !== "string" || + !/^\d+$/.test(value.storeDevice) || + typeof value.storeInode !== "string" || + !/^\d+$/.test(value.storeInode) || + (value.publicationState !== "ready" && value.publicationState !== "time-unsafe") || + !( + value.preservedLeaseStartedWallMs === null || isSafeInteger(value.preservedLeaseStartedWallMs) + ) || + !( + value.preservedLeaseExpiresWallMs === null || isSafeInteger(value.preservedLeaseExpiresWallMs) + ) || + !(value.preservedLeaseElapsedMs === null || isSafeInteger(value.preservedLeaseElapsedMs)) + ) { + throw new MalformedReceiptError(); + } + const hasStart = value.preservedLeaseStartedWallMs !== null; + const hasExpiry = value.preservedLeaseExpiresWallMs !== null; + const hasElapsed = value.preservedLeaseElapsedMs !== null; + if (hasStart !== hasExpiry || hasStart !== hasElapsed) throw new MalformedReceiptError(); + if ( + hasStart && + (value.preservedLeaseExpiresWallMs as number) - + (value.preservedLeaseStartedWallMs as number) !== + LAUNCH_READINESS_LEASE_MS + ) { + throw new MalformedReceiptError(); + } + if (hasElapsed && (value.preservedLeaseElapsedMs as number) > LAUNCH_READINESS_LEASE_MS) { + throw new MalformedReceiptError(); + } + return value as unknown as LaunchReadinessAuthority; +} + +function readLinuxBootId(): string | null { + try { + const value = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + return BOOT_ID_RE.test(value) ? value : null; + } catch { + return null; + } +} + +function readDarwinBootId(): string | null { + try { + const output = execFileSync("/usr/sbin/sysctl", ["-n", "kern.boottime"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + timeout: 2_000, + }); + const match = output.match(/sec\s*=\s*(\d+)\s*,\s*usec\s*=\s*(\d+)/); + if (!match) return null; + const value = `darwin:${match[1]}:${match[2]}`; + return BOOT_ID_RE.test(value) ? value : null; + } catch { + return null; + } +} + +export function readTrustedBootId(): string | null { + if (process.platform === "linux") return readLinuxBootId(); + if (process.platform === "darwin") return readDarwinBootId(); + return null; +} + +function currentUid(): number | null { + const value = process.getuid?.(); + return Number.isSafeInteger(value) && (value ?? -1) >= 0 ? (value as number) : null; +} + +function trustedCurrentUserHome(uid: number): string | null { + try { + const user = os.userInfo(); + if (user.uid !== uid || typeof user.homedir !== "string" || !path.isAbsolute(user.homedir)) { + return null; + } + return user.homedir; + } catch { + return null; + } +} + +function receiptKey(sandboxName: string): string { + return createHash("sha256").update(sandboxName, "utf8").digest("hex"); +} + +function buildBaseContext(options: LaunchReadinessStoreOptions): BaseContext { + const uid = (options.uid ?? currentUid)(); + const bootId = (options.bootId ?? readTrustedBootId)(); + const nowWallMs = (options.nowWallMs ?? Date.now)(); + const nowUptimeMs = (options.nowUptimeMs ?? (() => Math.floor(os.uptime() * 1_000)))(); + if ( + uid === null || + !Number.isSafeInteger(uid) || + uid < 0 || + !bootId || + !BOOT_ID_RE.test(bootId) || + !isSafeInteger(nowWallMs) || + !isSafeInteger(nowUptimeMs) + ) { + throw new UnsafeReceiptError(); + } + return { + uid, + nowWallMs, + nowUptimeMs, + bootId, + randomEpoch: options.randomEpoch ?? (() => randomBytes(32).toString("hex")), + runtimeAuthorityRoot: options.runtimeAuthorityRoot ?? null, + }; +} + +function buildStoreContext( + base: BaseContext, + sandboxName: string, + gatewayPort: number, + options: LaunchReadinessStoreOptions, +): StoreContext { + const { uid } = base; + const homeAuthority = options.home ?? trustedCurrentUserHome(uid); + if (!homeAuthority) throw new UnsafeReceiptError(); + const home = path.resolve(homeAuthority); + let homeStat: fs.Stats; + try { + homeStat = fs.lstatSync(home); + } catch { + throw new UnsafeReceiptError(); + } + if (!homeStat.isDirectory() || homeStat.isSymbolicLink() || homeStat.uid !== uid) { + throw new UnsafeReceiptError(); + } + const stateRoot = nemoclawStateRoot(home, gatewayPort); + const receiptDir = path.join(stateRoot, RECEIPT_DIRECTORY); + const key = receiptKey(sandboxName); + return { + ...base, + home, + stateRoot, + receiptDir, + receiptPath: path.join(receiptDir, `${key}.json`), + homeDevice: String(homeStat.dev), + homeInode: String(homeStat.ino), + }; +} + +function pathComponents(target: string): string[] { + if (!path.isAbsolute(target)) throw new UnsafeReceiptError(); + const parsed = path.parse(target); + const result = [parsed.root]; + let current = parsed.root; + for (const component of target.slice(parsed.root.length).split(path.sep).filter(Boolean)) { + current = path.join(current, component); + result.push(current); + } + return result; +} + +function validateDerivedRuntimeRoot( + candidate: string, + uid: number, +): { path: string; stat: fs.Stats } { + let canonical: string; + try { + canonical = fs.realpathSync.native(candidate); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + throw new MissingStoreError(); + } + throw new UnsafeReceiptError(); + } + if (!path.isAbsolute(canonical)) throw new UnsafeReceiptError(); + let sawUserOwner = false; + let finalStat: fs.Stats | null = null; + for (const component of pathComponents(canonical)) { + let stat: fs.Stats; + try { + stat = fs.lstatSync(component); + } catch { + throw new UnsafeReceiptError(); + } + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o022) !== 0) { + throw new UnsafeReceiptError(); + } + if (stat.uid === uid) { + sawUserOwner = true; + } else if (stat.uid !== 0 || sawUserOwner) { + throw new UnsafeReceiptError(); + } + finalStat = stat; + } + if (!finalStat || finalStat.uid !== uid || (finalStat.mode & 0o777) !== 0o700) { + throw new UnsafeReceiptError(); + } + return { path: canonical, stat: finalStat }; +} + +function derivedRuntimeRoot(uid: number): string { + if (process.platform === "linux") return `/run/user/${uid}`; + throw new UnsupportedAuthorityError(); +} + +function buildAuthorityContext(base: BaseContext, sandboxName: string): AuthorityContext { + const overridden = base.runtimeAuthorityRoot?.(); + const candidate = overridden ?? derivedRuntimeRoot(base.uid); + let runtimeRoot: string; + let runtimeStat: fs.Stats; + if (overridden !== null && overridden !== undefined) { + runtimeRoot = path.resolve(overridden); + try { + runtimeStat = fs.lstatSync(runtimeRoot); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + throw new MissingStoreError(); + } + throw new UnsafeReceiptError(); + } + if ( + !runtimeStat.isDirectory() || + runtimeStat.isSymbolicLink() || + runtimeStat.uid !== base.uid || + (runtimeStat.mode & 0o777) !== 0o700 + ) { + throw new UnsafeReceiptError(); + } + } else { + const verified = validateDerivedRuntimeRoot(candidate, base.uid); + runtimeRoot = verified.path; + runtimeStat = verified.stat; + } + const authorityDir = path.join(runtimeRoot, AUTHORITY_PRODUCT_DIRECTORY, AUTHORITY_DIRECTORY); + return { + ...base, + runtimeRoot, + authorityDir, + authorityPath: path.join(authorityDir, `${receiptKey(sandboxName)}.json`), + runtimeRootDevice: String(runtimeStat.dev), + runtimeRootInode: String(runtimeStat.ino), + }; +} + +function sameIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function assertSecureDirectoryStat(stat: fs.Stats, uid: number, privateDirectory: boolean): void { + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== uid || + (stat.mode & 0o022) !== 0 + ) { + throw new UnsafeReceiptError(); + } + if (privateDirectory && (stat.mode & 0o777) !== 0o700) throw new UnsafeReceiptError(); +} + +function ancestorPaths(root: string, target: string): string[] { + const relative = path.relative(root, target); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new UnsafeReceiptError(); + const paths = [root]; + let current = root; + for (const component of relative.split(path.sep).filter(Boolean)) { + current = path.join(current, component); + paths.push(current); + } + return paths; +} + +function fsyncSecureDirectoryPath(directoryPath: string, uid: number): void { + const flags = + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_DIRECTORY ?? 0); + let fd: number | null = null; + try { + fd = fs.openSync(directoryPath, flags); + assertSecureDirectoryStat(fs.fstatSync(fd), uid, false); + fs.fsyncSync(fd); + } catch { + throw new UnsafeReceiptError(); + } finally { + if (fd !== null) fs.closeSync(fd); + } +} + +function ensureSecureDirectory( + context: BaseContext, + root: string, + directoryPath: string, + create: boolean, + privateFrom = directoryPath, +): SecureDirectory { + const paths = ancestorPaths(root, directoryPath); + const ancestors: SecureDirectory["ancestors"] = []; + for (const [index, candidate] of paths.entries()) { + const privateDirectory = + candidate === directoryPath || + candidate.startsWith(`${privateFrom}${path.sep}`) || + candidate === privateFrom; + let stat: fs.Stats; + try { + stat = fs.lstatSync(candidate); + assertSecureDirectoryStat(stat, context.uid, privateDirectory); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT" && !create) { + throw new MissingStoreError(); + } + if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") { + if (error instanceof UnsafeReceiptError) throw error; + throw new UnsafeReceiptError(); + } + if (index === 0) throw new UnsafeReceiptError(); + try { + fs.mkdirSync(candidate, { mode: 0o700 }); + fs.chmodSync(candidate, 0o700); + stat = fs.lstatSync(candidate); + assertSecureDirectoryStat(stat, context.uid, privateDirectory); + fsyncSecureDirectoryPath(path.dirname(candidate), context.uid); + } catch { + throw new UnsafeReceiptError(); + } + } + ancestors.push({ path: candidate, stat, privateDirectory }); + } + + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const directoryOnly = fs.constants.O_DIRECTORY ?? 0; + let fd: number; + try { + fd = fs.openSync(directoryPath, fs.constants.O_RDONLY | noFollow | directoryOnly); + } catch { + throw new UnsafeReceiptError(); + } + try { + const descriptorStat = fs.fstatSync(fd); + const pathStat = fs.lstatSync(directoryPath); + assertSecureDirectoryStat(descriptorStat, context.uid, true); + assertSecureDirectoryStat(pathStat, context.uid, true); + if (!sameIdentity(descriptorStat, pathStat)) throw new UnsafeReceiptError(); + for (const ancestor of ancestors) { + const current = fs.lstatSync(ancestor.path); + assertSecureDirectoryStat(current, context.uid, ancestor.privateDirectory); + if (!sameIdentity(ancestor.stat, current)) throw new UnsafeReceiptError(); + } + return { fd, stat: descriptorStat, path: directoryPath, ancestors }; + } catch (error) { + fs.closeSync(fd); + if (error instanceof UnsafeReceiptError) throw error; + throw new UnsafeReceiptError(); + } +} + +function revalidateDirectory(context: BaseContext, directory: SecureDirectory): void { + for (const ancestor of directory.ancestors) { + const current = fs.lstatSync(ancestor.path); + assertSecureDirectoryStat(current, context.uid, ancestor.privateDirectory); + if (!sameIdentity(ancestor.stat, current)) throw new UnsafeReceiptError(); + } + const descriptorStat = fs.fstatSync(directory.fd); + const pathStat = fs.lstatSync(directory.path); + assertSecureDirectoryStat(descriptorStat, context.uid, true); + assertSecureDirectoryStat(pathStat, context.uid, true); + if (!sameIdentity(directory.stat, descriptorStat) || !sameIdentity(directory.stat, pathStat)) { + throw new UnsafeReceiptError(); + } +} + +function assertSecureReceiptStat(stat: fs.Stats, uid: number): void { + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.uid !== uid || + (stat.mode & 0o777) !== 0o600 || + stat.nlink !== 1 || + stat.size < 1 || + stat.size > LAUNCH_READINESS_MAX_BYTES + ) { + throw new UnsafeReceiptError(); + } +} + +function readSecureFileAtPath( + context: BaseContext, + directory: SecureDirectory, + filePath: string, +): string { + revalidateDirectory(context, directory); + const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0); + let fd: number; + try { + fd = fs.openSync(filePath, flags); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + throw new MissingStoreError(); + } + throw new UnsafeReceiptError(); + } + try { + const before = fs.fstatSync(fd); + assertSecureReceiptStat(before, context.uid); + const buffer = Buffer.alloc(LAUNCH_READINESS_MAX_BYTES + 1); + let total = 0; + while (total < buffer.length) { + const count = fs.readSync(fd, buffer, total, buffer.length - total, null); + if (count === 0) break; + total += count; + } + if (total > LAUNCH_READINESS_MAX_BYTES) throw new UnsafeReceiptError(); + const after = fs.fstatSync(fd); + const pathStat = fs.lstatSync(filePath); + assertSecureReceiptStat(after, context.uid); + assertSecureReceiptStat(pathStat, context.uid); + if ( + !sameIdentity(before, after) || + !sameIdentity(before, pathStat) || + before.size !== after.size || + total !== after.size + ) { + throw new UnsafeReceiptError(); + } + revalidateDirectory(context, directory); + return buffer.subarray(0, total).toString("utf8"); + } finally { + fs.closeSync(fd); + } +} + +function readRecordAtPath( + context: StoreContext, + directory: SecureDirectory, +): LaunchReadinessRecord { + return parseRecord(readSecureFileAtPath(context, directory, context.receiptPath)); +} + +function readAuthorityAtPath( + context: AuthorityContext, + directory: SecureDirectory, +): LaunchReadinessAuthority { + return parseAuthority(readSecureFileAtPath(context, directory, context.authorityPath)); +} + +function tempPath(filePath: string): string { + return path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${randomBytes(12).toString("hex")}.tmp`, + ); +} + +function proveWritable(context: BaseContext, directory: SecureDirectory, filePath: string): void { + revalidateDirectory(context, directory); + const candidate = tempPath(filePath); + let fd: number | null = null; + try { + fd = fs.openSync( + candidate, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + (fs.constants.O_NOFOLLOW ?? 0), + 0o600, + ); + fs.fchmodSync(fd, 0o600); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = null; + revalidateDirectory(context, directory); + fs.unlinkSync(candidate); + fs.fsyncSync(directory.fd); + } catch { + if (fd !== null) fs.closeSync(fd); + try { + fs.unlinkSync(candidate); + } catch { + // The store is already classified unsafe; cleanup is best effort. + } + throw new UnsafeReceiptError(); + } +} + +function writeSecureJson( + context: BaseContext, + directory: SecureDirectory, + filePath: string, + value: LaunchReadinessRecord | LaunchReadinessAuthority, + verify: (raw: string) => void, +): void { + const serialized = `${JSON.stringify(value)}\n`; + if (Buffer.byteLength(serialized) > LAUNCH_READINESS_MAX_BYTES) { + throw new UnsafeReceiptError(); + } + revalidateDirectory(context, directory); + const candidate = tempPath(filePath); + let fd: number | null = null; + try { + fd = fs.openSync( + candidate, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + (fs.constants.O_NOFOLLOW ?? 0), + 0o600, + ); + fs.fchmodSync(fd, 0o600); + const buffer = Buffer.from(serialized, "utf8"); + let written = 0; + while (written < buffer.length) written += fs.writeSync(fd, buffer, written); + fs.fsyncSync(fd); + const tempStat = fs.fstatSync(fd); + assertSecureReceiptStat(tempStat, context.uid); + fs.closeSync(fd); + fd = null; + revalidateDirectory(context, directory); + const tempPathStat = fs.lstatSync(candidate); + assertSecureReceiptStat(tempPathStat, context.uid); + if (!sameIdentity(tempStat, tempPathStat)) throw new UnsafeReceiptError(); + fs.renameSync(candidate, filePath); + fs.fsyncSync(directory.fd); + verify(readSecureFileAtPath(context, directory, filePath)); + } catch (error) { + if (fd !== null) fs.closeSync(fd); + try { + fs.unlinkSync(candidate); + } catch { + // Preserve the original store error. + } + if (error instanceof UnsafeReceiptError || error instanceof MalformedReceiptError) throw error; + throw new UnsafeReceiptError(); + } +} + +function writeRecord( + context: StoreContext, + directory: SecureDirectory, + record: LaunchReadinessRecord, +): void { + writeSecureJson(context, directory, context.receiptPath, record, (raw) => { + const published = parseRecord(raw); + if (JSON.stringify(published) !== JSON.stringify(record)) { + throw new UnsafeReceiptError(); + } + }); +} + +function authorityContextMatches( + authority: LaunchReadinessAuthority, + context: AuthorityContext, + directory: SecureDirectory, +): boolean { + return ( + authority.sandboxName.length > 0 && + authority.uid === context.uid && + authority.bootId === context.bootId && + authority.runtimeRootDevice === context.runtimeRootDevice && + authority.runtimeRootInode === context.runtimeRootInode && + authority.storeDevice === String(directory.stat.dev) && + authority.storeInode === String(directory.stat.ino) + ); +} + +function writeAuthority( + context: AuthorityContext, + directory: SecureDirectory, + authority: LaunchReadinessAuthority, +): LaunchReadinessAuthority { + writeSecureJson(context, directory, context.authorityPath, authority, (raw) => { + const published = parseAuthority(raw); + if ( + JSON.stringify(published) !== JSON.stringify(authority) || + !authorityContextMatches(published, context, directory) + ) { + throw new UnsafeReceiptError(); + } + }); + return authority; +} + +function ensureAuthorityDirectory(context: AuthorityContext, create: boolean): SecureDirectory { + return ensureSecureDirectory( + context, + context.runtimeRoot, + context.authorityDir, + create, + path.join(context.runtimeRoot, AUTHORITY_PRODUCT_DIRECTORY), + ); +} + +type AuthorityInspection = + | { kind: "missing"; context: AuthorityContext | null } + | { kind: "unsupported"; context: null } + | { kind: "present"; context: AuthorityContext; authority: LaunchReadinessAuthority } + | { kind: "unsafe"; context: AuthorityContext | null }; + +function inspectAuthority(base: BaseContext, sandboxName: string): AuthorityInspection { + let context: AuthorityContext; + let directory: SecureDirectory | null = null; + try { + context = buildAuthorityContext(base, sandboxName); + } catch (error) { + if (error instanceof UnsupportedAuthorityError) { + return { kind: "unsupported", context: null }; + } + return error instanceof MissingStoreError + ? { kind: "missing", context: null } + : { kind: "unsafe", context: null }; + } + try { + directory = ensureAuthorityDirectory(context, false); + const authority = readAuthorityAtPath(context, directory); + proveWritable(context, directory, context.authorityPath); + if (!authorityContextMatches(authority, context, directory)) { + return { kind: "unsafe", context }; + } + return { kind: "present", context, authority }; + } catch (error) { + return error instanceof MissingStoreError + ? { kind: "missing", context } + : { kind: "unsafe", context }; + } finally { + closeDirectory(directory); + } +} + +function rotateAuthority( + context: AuthorityContext, + sandboxName: string, + gatewayName: string, + gatewayPort: number, + prior: LaunchReadinessAuthority | null, + epochId: string, + publicationDisabled: boolean, +): LaunchReadinessAuthority { + if (!isEpoch(epochId)) throw new UnsafeReceiptError(); + const directory = ensureAuthorityDirectory(context, true); + try { + let publication = publicationDisabled + ? { timeline: null, state: "time-unsafe" as const } + : authorityPublicationTimeline(prior, context); + if (publication.state === "time-unsafe" && publication.timeline === null) { + const expires = context.nowWallMs + LAUNCH_READINESS_LEASE_MS; + if (!Number.isSafeInteger(expires)) throw new UnsafeReceiptError(); + publication = { + state: "time-unsafe", + timeline: { started: context.nowWallMs, expires, elapsed: 0 }, + }; + } + const authority: LaunchReadinessAuthority = { + schemaVersion: 1, + kind: "authority", + phase: "fence", + epochId, + sandboxName, + gatewayName, + gatewayPort, + observedWallMs: context.nowWallMs, + observedUptimeMs: context.nowUptimeMs, + bootId: context.bootId, + uid: context.uid, + runtimeRootDevice: context.runtimeRootDevice, + runtimeRootInode: context.runtimeRootInode, + storeDevice: String(directory.stat.dev), + storeInode: String(directory.stat.ino), + publicationState: publication.state, + preservedLeaseStartedWallMs: publication.timeline?.started ?? null, + preservedLeaseExpiresWallMs: publication.timeline?.expires ?? null, + preservedLeaseElapsedMs: publication.timeline?.elapsed ?? null, + }; + return writeAuthority(context, directory, authority); + } finally { + closeDirectory(directory); + } +} + +function recordContextMatches( + record: LaunchReadinessRecord, + context: StoreContext, + storeStat: fs.Stats, + gatewayName: string, + gatewayPort: number, +): boolean { + return ( + record.uid === context.uid && + record.bootId === context.bootId && + record.homeDevice === context.homeDevice && + record.homeInode === context.homeInode && + record.storeDevice === String(storeStat.dev) && + record.storeInode === String(storeStat.ino) && + record.gatewayName === gatewayName && + record.gatewayPort === gatewayPort + ); +} + +function validateLeaseTime( + lease: LaunchReadinessLease, + context: StoreContext, + storeStat: fs.Stats, + gatewayName: string, + gatewayPort: number, +): "valid" | "expired" | "identity" | "malformed" { + if (lease.leaseExpiresWallMs - lease.leaseStartedWallMs !== LAUNCH_READINESS_LEASE_MS) { + return "malformed"; + } + if ( + lease.publishedWallMs < lease.leaseStartedWallMs || + lease.publishedWallMs > lease.leaseExpiresWallMs || + lease.elapsedAtPublicationMs >= LAUNCH_READINESS_LEASE_MS || + lease.elapsedAtPublicationMs < lease.publishedWallMs - lease.leaseStartedWallMs || + context.nowWallMs < lease.leaseStartedWallMs || + context.nowWallMs < lease.publishedWallMs + ) { + return "malformed"; + } + const wallElapsed = context.nowWallMs - lease.leaseStartedWallMs; + if (context.nowWallMs >= lease.leaseExpiresWallMs || wallElapsed > LAUNCH_READINESS_LEASE_MS) { + return "expired"; + } + if (!recordContextMatches(lease, context, storeStat, gatewayName, gatewayPort)) return "identity"; + if (context.nowUptimeMs < lease.publishedUptimeMs) return "malformed"; + const monotonicElapsed = + lease.elapsedAtPublicationMs + (context.nowUptimeMs - lease.publishedUptimeMs); + if ( + !Number.isSafeInteger(monotonicElapsed) || + monotonicElapsed < 0 || + monotonicElapsed >= LAUNCH_READINESS_LEASE_MS + ) { + return monotonicElapsed >= LAUNCH_READINESS_LEASE_MS ? "expired" : "malformed"; + } + return "valid"; +} + +function closeDirectory(directory: SecureDirectory | null): void { + if (directory) fs.closeSync(directory.fd); +} + +export function readLaunchReadinessLease( + sandboxName: string, + gatewayName: string, + gatewayPort: number, + options: LaunchReadinessStoreOptions = {}, +): LaunchReadinessLeaseRead { + let context: StoreContext | null = null; + let directory: SecureDirectory | null = null; + let authorityDirectory: SecureDirectory | null = null; + try { + const base = buildBaseContext(options); + context = buildStoreContext(base, sandboxName, gatewayPort, options); + directory = ensureSecureDirectory(context, context.home, context.receiptDir, false); + const record = readRecordAtPath(context, directory); + if (record.kind !== "lease" || record.sandboxName !== sandboxName) return { kind: "missing" }; + if (record.gatewayName !== record.identity.gatewayName) return { kind: "malformed" }; + proveWritable(context, directory, context.receiptPath); + const time = validateLeaseTime(record, context, directory.stat, gatewayName, gatewayPort); + if (time === "malformed") return { kind: "malformed" }; + if (time === "expired") return { kind: "expired", lease: record }; + if (time === "identity") return { kind: "identity", lease: record }; + try { + const authorityContext = buildAuthorityContext(base, sandboxName); + authorityDirectory = ensureAuthorityDirectory(authorityContext, false); + const authority = readAuthorityAtPath(authorityContext, authorityDirectory); + proveWritable(authorityContext, authorityDirectory, authorityContext.authorityPath); + if ( + authority.phase !== "lease" || + authority.sandboxName !== sandboxName || + authority.epochId !== record.epochId || + authority.gatewayName !== gatewayName || + authority.gatewayPort !== gatewayPort || + authority.publicationState !== "ready" || + authority.observedWallMs !== record.publishedWallMs || + authority.observedUptimeMs !== record.publishedUptimeMs || + authority.preservedLeaseStartedWallMs !== record.leaseStartedWallMs || + authority.preservedLeaseExpiresWallMs !== record.leaseExpiresWallMs || + authority.preservedLeaseElapsedMs !== record.elapsedAtPublicationMs || + !authorityContextMatches(authority, authorityContext, authorityDirectory) + ) { + return { kind: "identity", lease: record }; + } + } catch (error) { + if (error instanceof MissingStoreError) return { kind: "identity", lease: record }; + throw new UnsafeReceiptError(); + } + return { kind: "valid", lease: record }; + } catch (error) { + if (error instanceof MissingStoreError) return { kind: "missing" }; + if (error instanceof MalformedReceiptError) return { kind: "malformed" }; + return { kind: "unsafe" }; + } finally { + closeDirectory(authorityDirectory); + closeDirectory(directory); + } +} + +function recordTimeline( + record: LaunchReadinessRecord | LaunchReadinessAuthority | null, +): { started: number; expires: number; elapsed: number } | null { + const started = + record?.kind === "lease" ? record.leaseStartedWallMs : record?.preservedLeaseStartedWallMs; + const expires = + record?.kind === "lease" ? record.leaseExpiresWallMs : record?.preservedLeaseExpiresWallMs; + const elapsed = + record?.kind === "lease" ? record.elapsedAtPublicationMs : record?.preservedLeaseElapsedMs; + if ( + started === null || + started === undefined || + expires === null || + expires === undefined || + elapsed === null || + elapsed === undefined || + elapsed > LAUNCH_READINESS_LEASE_MS || + expires - started !== LAUNCH_READINESS_LEASE_MS + ) { + return null; + } + return { started, expires, elapsed }; +} + +function authorityPublicationTimeline( + authority: LaunchReadinessAuthority | null, + context: BaseContext, +): { + timeline: { started: number; expires: number; elapsed: number } | null; + state: "ready" | "time-unsafe"; +} { + const timeline = recordTimeline(authority); + if (!authority) return { timeline: null, state: "ready" }; + if (authority.publicationState === "time-unsafe") { + if (!timeline) return { timeline: null, state: "time-unsafe" }; + if ( + authority.bootId !== context.bootId || + context.nowWallMs < timeline.started || + context.nowWallMs < authority.observedWallMs || + context.nowUptimeMs < authority.observedUptimeMs + ) { + return { timeline, state: "time-unsafe" }; + } + const monotonicElapsed = timeline.elapsed + (context.nowUptimeMs - authority.observedUptimeMs); + const wallElapsed = context.nowWallMs - timeline.started; + if (!Number.isSafeInteger(monotonicElapsed) || !Number.isSafeInteger(wallElapsed)) { + return { timeline, state: "time-unsafe" }; + } + const elapsed = Math.min(LAUNCH_READINESS_LEASE_MS, monotonicElapsed); + return monotonicElapsed >= LAUNCH_READINESS_LEASE_MS && + wallElapsed >= LAUNCH_READINESS_LEASE_MS && + context.nowWallMs >= timeline.expires + ? { timeline: null, state: "ready" } + : { timeline: { ...timeline, elapsed }, state: "time-unsafe" }; + } + if (!timeline) return { timeline: null, state: "ready" }; + if ( + authority.bootId !== context.bootId || + context.nowWallMs < timeline.started || + context.nowWallMs < authority.observedWallMs || + context.nowUptimeMs < authority.observedUptimeMs + ) { + return { timeline, state: "time-unsafe" }; + } + const elapsed = Math.max( + timeline.elapsed + (context.nowUptimeMs - authority.observedUptimeMs), + context.nowWallMs - timeline.started, + ); + if ( + !Number.isSafeInteger(elapsed) || + elapsed >= LAUNCH_READINESS_LEASE_MS || + context.nowWallMs >= timeline.expires + ) { + return { timeline: null, state: "ready" }; + } + return { timeline: { ...timeline, elapsed }, state: "ready" }; +} + +type PersistentInspection = { kind: "missing" } | { kind: "present" } | { kind: "unsafe" }; + +function inspectPersistentStore( + base: BaseContext, + sandboxName: string, + gatewayPort: number, + options: LaunchReadinessStoreOptions, +): PersistentInspection { + let directory: SecureDirectory | null = null; + try { + const context = buildStoreContext(base, sandboxName, gatewayPort, options); + directory = ensureSecureDirectory(context, context.home, context.receiptDir, false); + try { + readRecordAtPath(context, directory); + return { kind: "present" }; + } catch (error) { + if (error instanceof MissingStoreError) return { kind: "missing" }; + if (error instanceof MalformedReceiptError) return { kind: "present" }; + return { kind: "unsafe" }; + } + } catch (error) { + return error instanceof MissingStoreError ? { kind: "missing" } : { kind: "unsafe" }; + } finally { + closeDirectory(directory); + } +} + +/** + * Revalidate the sandbox-global runtime epoch immediately before recovery. + * The caller must hold the sandbox lifecycle lock followed by the owning + * gateway route lock for this check and the complete mutation window. + */ +export function checkLaunchReadinessMutationAuthority( + sandboxName: string, + gatewayName: string, + gatewayPort: number, + expectedEpochId: string | null, + options: LaunchReadinessStoreOptions = {}, +): LaunchReadinessMutationAuthorityCheck { + if ( + !gatewayName || + gatewayName.length > 256 || + !Number.isInteger(gatewayPort) || + gatewayPort < 1 || + gatewayPort > 65535 || + (expectedEpochId !== null && !isEpoch(expectedEpochId)) + ) { + return "unsafe"; + } + try { + const base = buildBaseContext(options); + const inspection = inspectAuthority(base, sandboxName); + if (expectedEpochId === null) { + if (inspection.kind === "unsupported") return "current"; + const persistent = inspectPersistentStore(base, sandboxName, gatewayPort, options); + if (inspection.kind === "unsafe" || persistent.kind === "unsafe") return "unsafe"; + return inspection.kind === "missing" && persistent.kind === "missing" ? "current" : "changed"; + } + if (inspection.kind === "unsupported") return "changed"; + if (inspection.kind === "unsafe") return "unsafe"; + if (inspection.kind === "missing") return "changed"; + const { authority } = inspection; + return authority.phase === "fence" && + authority.epochId === expectedEpochId && + authority.sandboxName === sandboxName && + authority.gatewayName === gatewayName && + authority.gatewayPort === gatewayPort + ? "current" + : "changed"; + } catch { + return "unsafe"; + } +} + +/** + * Replace any prior lease with a durable random publication epoch. + * + * The caller must hold the sandbox lifecycle lock followed by the owning + * gateway route lock. After the caller releases those locks following the + * initial rotation, it must reacquire both through the mutation gate, + * revalidate this epoch, and hold them for the complete preflight and final + * publication. + */ +export function fenceLaunchReadinessLease( + sandboxName: string, + gatewayName: string, + gatewayPort: number, + options: LaunchReadinessStoreOptions = {}, +): LaunchReadinessFence { + if ( + !gatewayName || + gatewayName.length > 256 || + !Number.isInteger(gatewayPort) || + gatewayPort < 1 || + gatewayPort > 65535 + ) { + throw new LaunchReadinessFenceError(true, false); + } + const base = buildBaseContext(options); + const authorityInspection = inspectAuthority(base, sandboxName); + if (authorityInspection.kind === "unsupported") { + throw new LaunchReadinessFenceError(false, false, true); + } + const persistentInspection = inspectPersistentStore(base, sandboxName, gatewayPort, options); + const epochId = base.randomEpoch(); + if (!isEpoch(epochId)) throw new LaunchReadinessFenceError(true, false); + + let authorityContext: AuthorityContext; + try { + authorityContext = authorityInspection.context ?? buildAuthorityContext(base, sandboxName); + rotateAuthority( + authorityContext, + sandboxName, + gatewayName, + gatewayPort, + authorityInspection.kind === "present" ? authorityInspection.authority : null, + epochId, + authorityInspection.kind === "unsafe", + ); + } catch { + const blocksRecovery = + authorityInspection.kind !== "missing" || persistentInspection.kind !== "missing"; + throw new LaunchReadinessFenceError(blocksRecovery, false); + } + + let directory: SecureDirectory | null = null; + try { + const context = buildStoreContext(base, sandboxName, gatewayPort, options); + directory = ensureSecureDirectory(context, context.home, context.receiptDir, true); + let existing: LaunchReadinessRecord | null = null; + try { + existing = readRecordAtPath(context, directory); + if (existing.sandboxName !== sandboxName) existing = null; + } catch (error) { + if (!(error instanceof MissingStoreError) && !(error instanceof MalformedReceiptError)) { + throw error; + } + } + const currentAuthority = inspectAuthority(base, sandboxName); + if ( + currentAuthority.kind !== "present" || + currentAuthority.authority.epochId !== epochId || + currentAuthority.authority.gatewayName !== gatewayName || + currentAuthority.authority.gatewayPort !== gatewayPort + ) { + throw new UnsafeReceiptError(); + } + const authorityTimeline = authorityPublicationTimeline(currentAuthority.authority, base); + const fence: LaunchReadinessFence = { + schemaVersion: LAUNCH_READINESS_SCHEMA_VERSION, + kind: "fence", + epochId, + sandboxName, + fencedWallMs: context.nowWallMs, + fencedUptimeMs: context.nowUptimeMs, + bootId: context.bootId, + uid: context.uid, + homeDevice: context.homeDevice, + homeInode: context.homeInode, + storeDevice: String(directory.stat.dev), + storeInode: String(directory.stat.ino), + gatewayName, + gatewayPort, + publicationState: authorityTimeline.state, + preservedLeaseStartedWallMs: authorityTimeline.timeline?.started ?? null, + preservedLeaseExpiresWallMs: authorityTimeline.timeline?.expires ?? null, + preservedLeaseElapsedMs: authorityTimeline.timeline?.elapsed ?? null, + }; + writeRecord(context, directory, fence); + return fence; + } catch { + throw new LaunchReadinessFenceError(false, true); + } finally { + closeDirectory(directory); + } +} + +/** Publish only when the exact fence epoch is still authoritative. */ +export function publishLaunchReadinessLease( + sandboxName: string, + gatewayName: string, + gatewayPort: number, + expectedEpochId: string, + identity: LaunchReadinessIdentity, + options: LaunchReadinessStoreOptions = {}, +): LaunchReadinessLease { + let directory: SecureDirectory | null = null; + let authorityDirectory: SecureDirectory | null = null; + try { + const base = buildBaseContext(options); + const context = buildStoreContext(base, sandboxName, gatewayPort, options); + directory = ensureSecureDirectory(context, context.home, context.receiptDir, false); + const record = readRecordAtPath(context, directory); + if ( + record.kind !== "fence" || + record.sandboxName !== sandboxName || + record.epochId !== expectedEpochId || + !recordContextMatches(record, context, directory.stat, gatewayName, gatewayPort) + ) { + throw new Error("Launch readiness publication authority changed."); + } + if (record.publicationState !== "ready") { + throw new Error( + "Launch readiness publication is disabled while authority or clock history is unsafe.", + ); + } + const authorityContext = buildAuthorityContext(base, sandboxName); + authorityDirectory = ensureAuthorityDirectory(authorityContext, false); + const authority = readAuthorityAtPath(authorityContext, authorityDirectory); + if ( + authority.phase !== "fence" || + authority.epochId !== expectedEpochId || + authority.sandboxName !== sandboxName || + authority.gatewayName !== gatewayName || + authority.gatewayPort !== gatewayPort || + !authorityContextMatches(authority, authorityContext, authorityDirectory) + ) { + throw new Error("Launch readiness publication authority changed."); + } + proveWritable(authorityContext, authorityDirectory, authorityContext.authorityPath); + const publication = authorityPublicationTimeline(authority, base); + if (publication.state !== "ready") { + throw new Error("Launch readiness publication time is unsafe."); + } + const timeline = publication.timeline; + const leaseStartedWallMs = timeline?.started ?? context.nowWallMs; + const leaseExpiresWallMs = timeline?.expires ?? context.nowWallMs + LAUNCH_READINESS_LEASE_MS; + const elapsedAtPublicationMs = timeline?.elapsed ?? 0; + if (!Number.isSafeInteger(leaseExpiresWallMs)) throw new UnsafeReceiptError(); + const lease: LaunchReadinessLease = { + schemaVersion: LAUNCH_READINESS_SCHEMA_VERSION, + kind: "lease", + epochId: record.epochId, + sandboxName, + leaseStartedWallMs, + leaseExpiresWallMs, + elapsedAtPublicationMs, + publishedWallMs: context.nowWallMs, + publishedUptimeMs: context.nowUptimeMs, + bootId: context.bootId, + uid: context.uid, + homeDevice: context.homeDevice, + homeInode: context.homeInode, + storeDevice: String(directory.stat.dev), + storeInode: String(directory.stat.ino), + gatewayName, + gatewayPort, + identity, + }; + if (validateLeaseTime(lease, context, directory.stat, gatewayName, gatewayPort) !== "valid") { + throw new Error("Launch readiness lease time envelope is no longer valid."); + } + const leaseAuthority: LaunchReadinessAuthority = { + ...authority, + phase: "lease", + observedWallMs: context.nowWallMs, + observedUptimeMs: context.nowUptimeMs, + publicationState: "ready", + preservedLeaseStartedWallMs: leaseStartedWallMs, + preservedLeaseExpiresWallMs: leaseExpiresWallMs, + preservedLeaseElapsedMs: elapsedAtPublicationMs, + }; + writeAuthority(authorityContext, authorityDirectory, leaseAuthority); + writeRecord(context, directory, lease); + const finalAuthority = readAuthorityAtPath(authorityContext, authorityDirectory); + if ( + finalAuthority.epochId !== expectedEpochId || + finalAuthority.phase !== "lease" || + finalAuthority.observedWallMs !== lease.publishedWallMs || + finalAuthority.observedUptimeMs !== lease.publishedUptimeMs || + finalAuthority.gatewayName !== gatewayName || + finalAuthority.gatewayPort !== gatewayPort || + finalAuthority.preservedLeaseStartedWallMs !== lease.leaseStartedWallMs || + finalAuthority.preservedLeaseExpiresWallMs !== lease.leaseExpiresWallMs || + finalAuthority.preservedLeaseElapsedMs !== lease.elapsedAtPublicationMs || + !authorityContextMatches(finalAuthority, authorityContext, authorityDirectory) + ) { + throw new Error("Launch readiness publication authority changed."); + } + return lease; + } finally { + closeDirectory(authorityDirectory); + closeDirectory(directory); + } +} + +export function launchReadinessReceiptPath( + sandboxName: string, + gatewayPort: number, + home: string, +): string { + return path.join( + nemoclawStateRoot(path.resolve(home), gatewayPort), + RECEIPT_DIRECTORY, + `${receiptKey(sandboxName)}.json`, + ); +} + +export function launchReadinessAuthorityPath(sandboxName: string, runtimeRoot: string): string { + return path.join( + path.resolve(runtimeRoot), + AUTHORITY_PRODUCT_DIRECTORY, + AUTHORITY_DIRECTORY, + `${receiptKey(sandboxName)}.json`, + ); +} diff --git a/test/cli/connect-recovery-settle.test.ts b/test/cli/connect-recovery-settle.test.ts index 457e231a689..1d91421c2a4 100644 --- a/test/cli/connect-recovery-settle.test.ts +++ b/test/cli/connect-recovery-settle.test.ts @@ -12,6 +12,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { launchReadinessRegistryFixture } from "../helpers/launch-readiness-fixture"; import { runWithEnv, writeSandboxRegistry } from "./helpers"; const DECODE_SANDBOX_EXEC_COMMAND_LINES = [ @@ -22,13 +23,14 @@ const DECODE_SANDBOX_EXEC_COMMAND_LINES = [ describe("CLI dispatch", () => { it("fails probe-only when the authenticated settle probe detects a listener wedge (#4710)", () => { + const sandboxName = "probe-wedge-alpha"; const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-wedge-")); const localBin = path.join(home, "bin"); const markerFile = path.join(home, "openshell-calls"); const stateFile = path.join(home, "probe-state"); const readyCountFile = path.join(home, "ready-count"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); + writeSandboxRegistry(home, sandboxName, launchReadinessRegistryFixture()); fs.writeFileSync(stateFile, "stopped"); fs.writeFileSync( path.join(localBin, "openshell"), @@ -40,19 +42,19 @@ describe("CLI dispatch", () => { `ready_count_file=${JSON.stringify(readyCountFile)}`, 'printf \'%s\\n\' "$*" >> "$marker_file"', 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'alpha Ready'", + ` echo '${sandboxName} Ready'`, " exit 0", "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + `if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "\${!#}" = "${sandboxName}" ]; then`, " echo 'Sandbox:'", " echo", " echo ' Id: abc'", - " echo ' Name: alpha'", + ` echo ' Name: ${sandboxName}'`, " echo ' Namespace: openshell'", " echo ' Phase: Ready'", " exit 0", "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', + `if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [[ "$*" == *"--name ${sandboxName}"* ]]; then`, ' cmd="$8"', ' cmd="$(decode_sandbox_exec_cmd "$cmd")"', ' case "$cmd" in', @@ -100,7 +102,7 @@ describe("CLI dispatch", () => { 'printf \'docker %s\\n\' "$*" >> "$marker_file"', 'if [ "$1" = "info" ]; then echo "24.0.0"; exit 0; fi', 'if [ "$1" = "ps" ]; then', - " printf 'container-id\\topenshell-alpha\\n'", + ` printf 'container-id\\topenshell-${sandboxName}\\n'`, " exit 0", "fi", 'if [[ "$*" == *"--env LD_PRELOAD="* ]] && [[ "$*" == *"--env PYTHONPATH="* ]] && [[ "$*" == *"--user root container-id /usr/local/bin/nemoclaw-gateway-control recover "* ]]; then', @@ -123,7 +125,7 @@ describe("CLI dispatch", () => { { mode: 0o755 }, ); - const r = runWithEnv("alpha connect --probe-only", { + const r = runWithEnv(`${sandboxName} connect --probe-only`, { HOME: home, PATH: `${localBin}:${process.env.PATH || ""}`, NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS: "3", @@ -133,7 +135,7 @@ describe("CLI dispatch", () => { expect(r.code).toBe(1); expect(r.out).toContain( - "Probe failed: OpenClaw gateway is not running in 'alpha' and automatic recovery failed.", + `Probe failed: OpenClaw gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); expect(r.out).toContain("#4710 wedge signature"); expect(r.out).toContain("config change requires gateway restart (plugins.installs)"); diff --git a/test/cli/connect-recovery.test.ts b/test/cli/connect-recovery.test.ts index 794b9c65eb5..3a85fafdabb 100644 --- a/test/cli/connect-recovery.test.ts +++ b/test/cli/connect-recovery.test.ts @@ -7,6 +7,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { + LAUNCH_READINESS_FIXTURE_POLICY, + launchReadinessRegistryFixture, +} from "../helpers/launch-readiness-fixture"; import { nonWslPlatformNodeOptions } from "../helpers/platform-override-node-options"; import { runWithEnv, @@ -24,6 +28,25 @@ type GatewayControlDockerStubOptions = { recoveryStatus?: number; }; +const launchReadinessObservationStubLines = [ + 'if [ "$1" = "policy" ] && [ "$2" = "get" ]; then', + ` printf '%b' ${JSON.stringify(LAUNCH_READINESS_FIXTURE_POLICY)}`, + " exit 0", + "fi", + 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', + " printf '%s\\n' 'Gateway inference:' ' Provider: nvidia-prod' ' Model: test-model'", + " exit 0", + "fi", +]; + +const expectedProbeOnlyExitCode = process.platform === "darwin" ? 1 : 0; +const PLATFORM_EVIDENCE_UNAVAILABLE = "launch-readiness evidence is unavailable on this platform"; + +function expectProbeOnlyPublicationOutcome(result: { code: number; out: string }): void { + expect(result.code, result.out).toBe(expectedProbeOnlyExitCode); + expect(result.out.includes(PLATFORM_EVIDENCE_UNAVAILABLE)).toBe(process.platform === "darwin"); +} + function writeGatewayControlDockerStub( localBin: string, { @@ -198,7 +221,7 @@ describe("CLI connect recovery process contracts", () => { const sshMarkerFile = path.join(home, "ssh-calls"); const stateFile = path.join(home, "probe-state"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); + writeSandboxRegistry(home, launchReadinessRegistryFixture()); fs.writeFileSync(stateFile, "stopped"); fs.writeFileSync( path.join(localBin, "openshell"), @@ -218,22 +241,17 @@ describe("CLI connect recovery process contracts", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', ' if [[ "$*" == *"inference.local/v1/models"* ]]; then echo "OK 200"; exit 0; fi', - ' case "$cmd" in', - ' *"inference.local/v1/models"*)', - " echo 'OK 200'", - " exit 0", - " ;;", - " *'curl -so'*)", - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', - " exit 0", - " ;;", - " esac", + ' if [[ "$*" == *"NEMOCLAW_AGENT_SMOKE_EXIT"* ]]; then echo "NEMOCLAW_AGENT_SMOKE_BEGIN"; echo "NEMOCLAW_AGENT_SMOKE_EXIT:0"; exit 0; fi', + ' if [[ "$*" == *"curl -so"* ]]; then', + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', + " exit 0", + " fi", "fi", 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', 'if [ "$1" = "forward" ]; then exit 99; fi', + ...launchReadinessObservationStubLines, "exit 0", ].join("\n"), { mode: 0o755 }, @@ -255,7 +273,7 @@ describe("CLI connect recovery process contracts", () => { PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(result.code, result.out).toBe(0); + expectProbeOnlyPublicationOutcome(result); expect(result.out).toContain(expectedOutput); expect(result.out).not.toContain(unexpectedOutput); const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); @@ -284,7 +302,7 @@ describe("CLI connect recovery process contracts", () => { const sshCalls = path.join(home, "ssh-calls"); const stateFile = path.join(home, "probe-state"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); + writeSandboxRegistry(home, launchReadinessRegistryFixture()); fs.writeFileSync(stateFile, "stopped"); fs.writeFileSync( path.join(localBin, "openshell"), @@ -304,9 +322,9 @@ describe("CLI connect recovery process contracts", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', ' if [[ "$*" == *"inference.local/v1/models"* ]]; then echo "OK 200"; exit 0; fi', - ' if [[ "$cmd" == *"curl -so"* ]]; then', + ' if [[ "$*" == *"NEMOCLAW_AGENT_SMOKE_EXIT"* ]]; then echo "NEMOCLAW_AGENT_SMOKE_BEGIN"; echo "NEMOCLAW_AGENT_SMOKE_EXIT:0"; exit 0; fi', + ' if [[ "$*" == *"curl -so"* ]]; then', " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', " exit 0", @@ -314,6 +332,7 @@ describe("CLI connect recovery process contracts", () => { "fi", 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', 'if [ "$1" = "forward" ]; then exit 99; fi', + ...launchReadinessObservationStubLines, "exit 0", ].join("\n"), { mode: 0o755 }, @@ -355,7 +374,7 @@ describe("CLI connect recovery process contracts", () => { const sshCalls = path.join(home, "ssh-calls"); const stateFile = path.join(home, "probe-state"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home, { agent: "hermes" }); + writeSandboxRegistry(home, { ...launchReadinessRegistryFixture(), agent: "hermes" }); fs.writeFileSync(stateFile, "stopped"); fs.writeFileSync( path.join(localBin, "openshell"), @@ -375,9 +394,9 @@ describe("CLI connect recovery process contracts", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', ' if [[ "$*" == *"inference.local/v1/models"* ]]; then echo "OK 200"; exit 0; fi', - ' if [[ "$cmd" == *"curl -so"* ]]; then', + ' if [[ "$*" == *"NEMOCLAW_AGENT_SMOKE_EXIT"* ]]; then echo "NEMOCLAW_AGENT_SMOKE_BEGIN"; echo "NEMOCLAW_AGENT_SMOKE_EXIT:0"; exit 0; fi', + ' if [[ "$*" == *"curl -so"* ]]; then', " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', " exit 0", @@ -389,6 +408,7 @@ describe("CLI connect recovery process contracts", () => { "fi", 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then { echo "alpha 127.0.0.1 18789 12345 running"; echo "alpha 127.0.0.1 8642 12346 running"; }; exit 0; fi', 'if [ "$1" = "forward" ]; then exit 99; fi', + ...launchReadinessObservationStubLines, "exit 0", ].join("\n"), { mode: 0o755 }, @@ -404,7 +424,7 @@ describe("CLI connect recovery process contracts", () => { PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(result.code, result.out).toBe(0); + expectProbeOnlyPublicationOutcome(result); expect(result.out).toContain("Probe complete: recovered Hermes Agent gateway"); const openshellLog = fs.readFileSync(openshellCalls, "utf8"); expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); diff --git a/test/cli/connect-terminal-agent.test.ts b/test/cli/connect-terminal-agent.test.ts index cfb1fc69bd2..f3c7ff36db8 100644 --- a/test/cli/connect-terminal-agent.test.ts +++ b/test/cli/connect-terminal-agent.test.ts @@ -6,8 +6,11 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { launchReadinessRegistryFixture } from "../helpers/launch-readiness-fixture"; import { runWithEnv, writeSandboxRegistry } from "./helpers"; +const PLATFORM_EVIDENCE_UNAVAILABLE = "launch-readiness evidence is unavailable on this platform"; + describe("CLI dispatch for terminal agents", () => { it("connect --probe-only runs terminal-agent smoke checks without gateway recovery", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-terminal-")); @@ -15,6 +18,7 @@ describe("CLI dispatch for terminal agents", () => { const markerFile = path.join(home, "openshell-calls"); fs.mkdirSync(localBin, { recursive: true }); writeSandboxRegistry(home, { + ...launchReadinessRegistryFixture(), agent: "langchain-deepagents-code", provider: "", model: "", @@ -29,6 +33,14 @@ describe("CLI dispatch for terminal agents", () => { " echo 'alpha Ready'", " exit 0", "fi", + 'if [ "$1" = "policy" ] && [ "$2" = "get" ]; then', + " printf '%s\\n' 'version: 1' 'network_policies:' ' fixture_api:' ' name: Fixture API' ' endpoints:' ' - host: example.com' ' port: 443' ' binaries:' ' - path: /usr/bin/curl'", + " exit 0", + "fi", + 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', + " printf '%s\\n' 'Gateway inference:' ' Not configured'", + " exit 0", + "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && { [ "$3" = "alpha" ] || [ "$5" = "alpha" ]; }; then', " echo 'Sandbox:'", " echo", @@ -58,7 +70,8 @@ describe("CLI dispatch for terminal agents", () => { PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(r.code).toBe(0); + expect(r.code).toBe(process.platform === "darwin" ? 1 : 0); + expect(r.out.includes(PLATFORM_EVIDENCE_UNAVAILABLE)).toBe(process.platform === "darwin"); expect(r.out).toContain("terminal smoke checks passed"); const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); expect(calls).toContain("sandbox get -g nemoclaw alpha"); diff --git a/test/cli/docker-outage.test.ts b/test/cli/docker-outage.test.ts index 9154aa617d7..b1839eff0d1 100644 --- a/test/cli/docker-outage.test.ts +++ b/test/cli/docker-outage.test.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { launchReadinessRegistryFixture } from "../helpers/launch-readiness-fixture"; import { runWithEnv, testTimeout, writeSandboxRegistry } from "./helpers"; import type { SandboxEntry } from "./helpers"; @@ -28,6 +29,7 @@ describe("Docker daemon outage classification (#4428)", () => { // The Docker-outage reclassification only applies to Docker-driver // sandboxes (#4428); record the driver so the gate matches. writeSandboxRegistry(home, "v053-baseline", { + ...launchReadinessRegistryFixture(), policies: ["npm"], openshellDriver: driver, } as unknown as Partial); diff --git a/test/cli/launch-routing.test.ts b/test/cli/launch-routing.test.ts index b60f1593566..b340c428414 100644 --- a/test/cli/launch-routing.test.ts +++ b/test/cli/launch-routing.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterAll, describe, expect, it } from "vitest"; +import { launchReadinessRegistryFixture } from "../helpers/launch-readiness-fixture"; import { run, runWithEnv, testTimeoutOptions, writeSandboxRegistry } from "./helpers"; const CALL_SEPARATOR = "--- openshell call ---"; @@ -47,7 +48,7 @@ function createLaunchHarness(prefix: string, agent: string): LaunchHarness { const callArgvFile = path.join(home, "openshell-call-argv"); const execArgvFile = path.join(home, "openshell-exec-argv"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home, { agent }); + writeSandboxRegistry(home, { ...launchReadinessRegistryFixture(), agent }); fs.writeFileSync( path.join(localBin, "openshell"), @@ -160,6 +161,9 @@ describe("CLI launch routing process contracts (#6006)", () => { expect(result.code).toBe(0); expect(result.out).toContain("launch "); expect(result.out).toContain("Connect to a sandbox and start its agent"); + expect(result.out.replace(/\s+/g, " ")).toContain( + "Validate a current launch-readiness lease or run the complete connect preflight", + ); expect(result.out).toContain("SANDBOXNAME"); }); @@ -181,24 +185,19 @@ describe("CLI launch routing process contracts (#6006)", () => { expect(result.code).toBe(1); expect(result.out).toContain( - "Sandbox 'alpha;echo pwned' is registered locally, but is not present in the live OpenShell gateway.", + "Sandbox 'alpha;echo pwned' is not registered in the local NemoClaw state.", ); // The token never reaches an in-sandbox command: no interactive exec ran. expect(harness.launchExecArgv()).toBeNull(); expect(harness.callLines().some((call) => call.includes("--tty"))).toBe(false); - // Every `openshell` call that carries the token carries it as one argv - // element. No call splits it into a second command, which is what a - // shell-interpolated (rather than argv-passed) sandbox name would do. + // Local registry rejection happens before any OpenShell command can + // receive the untrusted token. const tokenCalls = harness .callArgvs() .filter((argv) => argv.some((element) => element.includes("pwned"))); - expect(tokenCalls.length).toBeGreaterThan(0); - for (const argv of tokenCalls) { - expect(argv).toContain("alpha;echo pwned"); - expect(argv).not.toContain("echo"); - expect(argv).not.toContain("pwned"); - } + expect(tokenCalls).toEqual([]); + expect(harness.callLines()).toEqual([]); }, ); @@ -214,8 +213,8 @@ describe("CLI launch routing process contracts (#6006)", () => { const result = harness.runLaunch("launch alpha"); expect(result.code).toBe(1); - expect(result.out).toContain( - 'Cannot resolve an interactive command for unsupported agent "mystery-agent; echo pwned".', + expect(result.out).toMatch( + /(?:Cannot resolve an interactive command for unsupported agent "mystery-agent; echo pwned"\.|Launch readiness final validation failed due to config\.)/, ); expect(harness.launchExecArgv()).toBeNull(); expect(harness.callLines().some((call) => call.includes("--tty"))).toBe(false); diff --git a/test/e2e/README.md b/test/e2e/README.md index 75e302c2069..3fec394a661 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -285,6 +285,32 @@ npx tsx tools/e2e/workflow-plan.mts --summary >> "$GITHUB_STEP_SUMMARY" The workflow's `--ci-output` mode uses the same renderer for its job summary. The table includes the typed registry matrix, shared test matrix, three catalogue profile matrices, and retained workflow jobs. +## Launch-readiness locked-image acceptance + +Use the repository helper to test an existing OpenClaw sandbox without +rebuilding its locked image: + +```bash +scripts/test-launch-readiness-lease.sh +``` + +Run this helper on Linux after the sandbox's final durable home and state +volume is mounted and after final policy and network provisioning is complete. +The launch-readiness lease path that it validates is currently Linux-only. +The helper must run as the same numeric user that later runs `launch`, and that user must own the sandbox's NemoClaw state. +The host must provide that user a secure, independently writable OS runtime authority under `/run/user/`; do not redirect it with environment variables. +The host must provide the util-linux `script` command and GNU `timeout` command. + +The helper rebuilds the candidate CLI, runs `connect --probe-only`, and then +runs two `launch` sessions during the same fixed lease. +Each pseudo-terminal session sends a unique prompt, requires the exact reply, +sends `/exit`, and requires process exit status `0`. +The helper uses exact terminal behavior instead of a wall-clock pass threshold. +Deterministic unit tests separately prove selection of the complete preflight +and lease paths, stale-producer exclusion, the fixed time-unsafe quarantine, +refusal to recover when prior evidence cannot be durably fenced, and the named +performance stages. + ## Inactive Windows MXC OpenClaw qualification `windows-mxc-openclaw-process-container.test.ts` is an explicit local diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 46d33def1b4..8cb8c7f8f0c 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -42,7 +42,7 @@ import { fullE2eInferenceProbeEvidence, runFullE2eInferenceProbe, } from "./full-e2e-inference-probe.ts"; -import { runLaunchAgentTurn } from "./launch-agent-turn.ts"; +import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; import { bindApprovedPrBaseForBaseImageComparison } from "./pr-base-comparison.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; @@ -141,7 +141,7 @@ async function runOpenClawLaunchTurnAfterRecovery(input: { ); expect(recovery.exitCode, resultText(recovery)).toBe(0); - await runLaunchAgentTurn({ + await runLaunchReadinessLeaseTurns({ artifactName: "phase-4-openclaw-launch-turn", cliCommand: USE_PREINSTALLED_LAUNCHABLE ? "nemoclaw" : process.execPath, ...(!USE_PREINSTALLED_LAUNCHABLE ? { cliEntrypoint: CLI_ENTRYPOINT } : {}), diff --git a/test/e2e/live/hermes-e2e-phases.ts b/test/e2e/live/hermes-e2e-phases.ts index 640bc94a19e..3813ef4e55f 100644 --- a/test/e2e/live/hermes-e2e-phases.ts +++ b/test/e2e/live/hermes-e2e-phases.ts @@ -5,7 +5,7 @@ export const HERMES_E2E_PHASES = [ "prepare clean Hermes runner", "install and onboard Hermes sandbox", "validate sandbox layout, health, and skill activation", - "restart Hermes gateway, validate supervision, and launch a turn", + "restart Hermes gateway, validate supervision, and complete two launch turns", "exercise hosted and inference.local routes", "validate CLI manifest and locked-config behavior", "finalize Hermes sandbox resources", diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 3e309bfec3f..2c4e6a1102d 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -24,7 +24,7 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { assertHermesCliAdapterLiveContract, stripAnsi } from "./hermes-cli-adapter-live.ts"; import { HERMES_E2E_PHASES } from "./hermes-e2e-phases.ts"; import { assertHermesSkillLifecycle } from "./hermes-skill-lifecycle.ts"; -import { runLaunchAgentTurn } from "./launch-agent-turn.ts"; +import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; import { expectPackageDatabaseReadOnly } from "./package-database-read-only.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes"; @@ -599,7 +599,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(httpStatusOk(dashboardInternal.stdout)).toBe(true); } - progress.phase("restart Hermes gateway, validate supervision, and launch a turn"); + progress.phase("restart Hermes gateway, validate supervision, and complete two launch turns"); // Phase 5: host-mediated Hermes gateway restart. This validates the // runtime contract behind #2426 against a real OpenShell/Hermes sandbox: // The installed supervision tree controls the gateway process, direct @@ -1257,7 +1257,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(routingTopologyCaptures).toBe(2); await (process.platform === "linux" - ? runLaunchAgentTurn({ + ? runLaunchReadinessLeaseTurns({ artifactName: "phase-5-hermes-launch-turn-after-recovery", cliCommand: "nemoclaw", env, diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index dfc9a0942e6..6d7df608401 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; + import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -47,6 +49,22 @@ timeout --kill-after=5s 250s \ session_pid=$! exec 3>"$input" +capture_ready=0 +for _ in {1..100}; do + if [[ -f "$capture" ]]; then + capture_ready=1 + break + fi + if ! kill -0 "$session_pid" 2>/dev/null; then + break + fi + sleep 0.1 +done +if [[ "$capture_ready" != 1 ]]; then + echo "launch did not create a terminal capture" >&2 + exit 1 +fi + if [[ -n "$NEMOCLAW_LAUNCH_READY_TEXT" ]]; then ready_seen=0 for _ in {1..60}; do @@ -71,9 +89,23 @@ response_start="$(wc -c <"$capture")" printf '%s\r' "$NEMOCLAW_LAUNCH_PROMPT" >&3 reply_seen=0 +has_exact_reply() { + tail -c "+$((response_start + 1))" "$capture" \ + | sed -E $'s/\x1B\\[[0-?]*[ -\\/]*[@-~]//g' \ + | tr '\r' '\n' \ + | LC_ALL=C tr -d '\000-\010\013\014\016-\037\177' \ + | awk -v expected="$NEMOCLAW_LAUNCH_EXPECTED_REPLY" ' + { + line = $0 + sub(/^[[:space:]]+/, "", line) + sub(/[[:space:]]+$/, "", line) + if (line == expected) found = 1 + } + END { exit found ? 0 : 1 } + ' +} for _ in {1..180}; do - if grep -Fq -- "$NEMOCLAW_LAUNCH_EXPECTED_REPLY" \ - < <(tail -c "+$((response_start + 1))" "$capture"); then + if has_exact_reply; then reply_seen=1 break fi @@ -123,6 +155,9 @@ export interface LaunchAgentTurnOptions { readyText?: string; redactionValues: string[]; sandboxName: string; + expectedReply?: string; + prompt?: string; + beforeLaunchTurns?: () => Promise | void; } export async function runLaunchAgentTurn( @@ -138,8 +173,8 @@ export async function runLaunchAgentTurn( NEMOCLAW_LAUNCH_COMMAND: options.cliCommand, NEMOCLAW_LAUNCH_ENTRYPOINT: options.cliEntrypoint ?? "", NEMOCLAW_LAUNCH_EXIT_COMMAND: options.exitCommand ?? "", - NEMOCLAW_LAUNCH_EXPECTED_REPLY: EXPECTED_REPLY, - NEMOCLAW_LAUNCH_PROMPT: PROMPT, + NEMOCLAW_LAUNCH_EXPECTED_REPLY: options.expectedReply ?? EXPECTED_REPLY, + NEMOCLAW_LAUNCH_PROMPT: options.prompt ?? PROMPT, NEMOCLAW_LAUNCH_READY_TEXT: options.readyText ?? "", NEMOCLAW_LAUNCH_SANDBOX: options.sandboxName, TERM: "xterm-256color", @@ -152,3 +187,42 @@ export async function runLaunchAgentTurn( } return result; } + +function uniqueTurnContract(artifactName: string, ordinal: "FIRST" | "SECOND") { + const fragment = createHash("sha256") + .update(`${artifactName}:${ordinal}`) + .digest("hex") + .slice(0, 12); + const expectedReply = `NEMOCLAW_${fragment.toUpperCase()}_${ordinal}_OK`; + return { + expectedReply, + prompt: + `Join these four fragments with underscores and put only the result on its own line: ` + + `NEMOCLAW, ${fragment.toUpperCase()}, ${ordinal}, OK. Do not use tools.`, + }; +} + +export async function runLaunchReadinessLeaseTurns(options: LaunchAgentTurnOptions): Promise { + const probeArgs = options.cliEntrypoint + ? [options.cliEntrypoint, options.sandboxName, "connect", "--probe-only"] + : [options.sandboxName, "connect", "--probe-only"]; + const probe = await options.host.command(options.cliCommand, probeArgs, { + artifactName: `${options.artifactName}-probe`, + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 360_000, + }); + if (probe.exitCode !== 0) { + throw new Error(`launch readiness producer failed: ${resultText(probe)}`); + } + + await options.beforeLaunchTurns?.(); + + for (const ordinal of ["FIRST", "SECOND"] as const) { + await runLaunchAgentTurn({ + ...options, + artifactName: `${options.artifactName}-${ordinal.toLowerCase()}`, + ...uniqueTurnContract(options.artifactName, ordinal), + }); + } +} diff --git a/test/e2e/live/launch-readiness-lease-acceptance.test.ts b/test/e2e/live/launch-readiness-lease-acceptance.test.ts new file mode 100644 index 00000000000..2837b943fe6 --- /dev/null +++ b/test/e2e/live/launch-readiness-lease-acceptance.test.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, test } from "../fixtures/e2e-test.ts"; +import { CLI_ENTRYPOINT } from "../fixtures/paths.ts"; +import { readRegistrySandboxEntry } from "../fixtures/phases/index.ts"; +import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; + +const SANDBOX_NAME = process.env.NEMOCLAW_ACCEPTANCE_SANDBOX?.trim() ?? ""; + +test.runIf(process.platform === "linux" && SANDBOX_NAME.length > 0)( + "launch readiness locked-image acceptance", + { + timeout: 15 * 60_000, + meta: { + e2ePhases: [ + "verify the existing locked-image sandbox", + "produce launch-readiness evidence", + "complete two exact-reply PTY launch turns", + ], + }, + }, + async ({ host, progress }) => { + progress.phase("verify the existing locked-image sandbox"); + const entry = readRegistrySandboxEntry(SANDBOX_NAME); + expect(entry.agent).toBe("openclaw"); + expect(entry.workload).toMatchObject({ kind: "managed-image" }); + expect(typeof entry.workload).toBe("object"); + expect(entry.workload).not.toBeNull(); + const workload = entry.workload as Record; + expect(workload.reference).toMatch(/@sha256:[0-9a-f]{64}$/u); + expect(entry.imageTag).toBe(workload.reference); + progress.phase("produce launch-readiness evidence"); + await runLaunchReadinessLeaseTurns({ + artifactName: "launch-readiness-locked-image", + cliCommand: process.execPath, + cliEntrypoint: CLI_ENTRYPOINT, + env: process.env, + exitCommand: "/exit", + host, + readyText: "gateway connected | idle", + redactionValues: [], + sandboxName: SANDBOX_NAME, + beforeLaunchTurns: () => { + progress.phase("complete two exact-reply PTY launch turns"); + }, + }); + }, +); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 3260cec4e68..1a5172479f9 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -109,6 +109,16 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/launch-readiness-lease-acceptance.test.ts", + "fast": [ + "src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts", + "src/lib/actions/sandbox/launch-readiness.test.ts", + "src/lib/state/launch-readiness-lease.test.ts", + "test/e2e/support/launch-agent-turn.test.ts", + "test/launch-readiness-forward-observation.test.ts" + ] + }, { "live": "test/e2e/live/bootstrap-install-smoke.test.ts", "fast": [ diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index 002bd2d8dcc..f36a9b2427a 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -7,9 +7,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, it } from "vitest"; -import { LAUNCH_TURN_SCRIPT } from "../live/launch-agent-turn.ts"; +import { LAUNCH_TURN_SCRIPT, runLaunchReadinessLeaseTurns } from "../live/launch-agent-turn.ts"; -function runLaunchTurnFixture(exitStatus: number) { +function runLaunchTurnFixture(exitStatus: number, reply = "PONG", closeAfterReply = false) { const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-turn-")); const scriptStub = join(fixtureRoot, "script"); const sleepStub = join(fixtureRoot, "sleep"); @@ -26,13 +26,17 @@ for argument in "$@"; do done : >"$capture" IFS= read -r -d $'\r' _ -printf 'PONG\n' | tee "$capture" -IFS= read -r -d $'\r' exit_command -[[ "$exit_command" == "/exit" ]] +printf '%s\n' "$NEMOCLAW_FIXTURE_REPLY" | tee "$capture" +${ + closeAfterReply + ? "" + : String.raw`IFS= read -r -d $'\r' exit_command +[[ "$exit_command" == "/exit" ]]` +} exit ${exitStatus} `, ); - writeFileSync(sleepStub, "#!/bin/sh\n/bin/sleep 0.5\n"); + writeFileSync(sleepStub, '#!/bin/sh\nif [ "${1:-}" = "0.1" ]; then /bin/sleep 0.01; fi\n'); writeFileSync(timeoutStub, '#!/bin/sh\nshift 2\nexec "$@"\n'); chmodSync(scriptStub, 0o755); chmodSync(sleepStub, 0o755); @@ -44,8 +48,9 @@ exit ${exitStatus} ...process.env, NEMOCLAW_LAUNCH_COMMAND: "ignored", NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", + NEMOCLAW_LAUNCH_EXIT_COMMAND: closeAfterReply ? "" : "/exit", NEMOCLAW_LAUNCH_EXPECTED_REPLY: "PONG", + NEMOCLAW_FIXTURE_REPLY: reply, NEMOCLAW_LAUNCH_PROMPT: "prompt", NEMOCLAW_LAUNCH_READY_TEXT: "", NEMOCLAW_LAUNCH_SANDBOX: "sandbox", @@ -58,6 +63,54 @@ exit ${exitStatus} } } +it.runIf(process.platform === "linux")( + "runs producer then two distinct PTY launch turns under one lease (#8942)", + async () => { + const calls: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = []; + let launchPhaseStartedAtCallCount = -1; + const host = { + command: async (command: string, args: string[], options: { env?: NodeJS.ProcessEnv }) => { + calls.push({ command, args, env: options.env }); + return { + exitCode: 0, + signal: null, + stdout: args.includes("--probe-only") ? "Probe complete" : "NEMOCLAW_LAUNCH_TURN_OK", + stderr: "", + }; + }, + }; + + await runLaunchReadinessLeaseTurns({ + artifactName: "lease-turn", + cliCommand: "node", + cliEntrypoint: "/repo/bin/nemoclaw.js", + env: {}, + exitCommand: "/exit", + host: host as never, + readyText: "gateway connected | idle", + redactionValues: [], + sandboxName: "alpha", + beforeLaunchTurns: () => { + launchPhaseStartedAtCallCount = calls.length; + }, + }); + + expect(calls).toHaveLength(3); + expect(launchPhaseStartedAtCallCount).toBe(1); + expect(calls[0]).toMatchObject({ + command: "node", + args: ["/repo/bin/nemoclaw.js", "alpha", "connect", "--probe-only"], + }); + expect(calls[1]?.env?.NEMOCLAW_LAUNCH_EXPECTED_REPLY).not.toBe( + calls[2]?.env?.NEMOCLAW_LAUNCH_EXPECTED_REPLY, + ); + expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_EXIT_COMMAND)).toEqual([ + "/exit", + "/exit", + ]); + }, +); + it.runIf(process.platform !== "win32")( "waits for OpenClaw gateway readiness before sending the launch prompt (#7230)", () => { @@ -130,6 +183,23 @@ it.runIf(process.platform !== "win32")( }, ); +it.runIf(process.platform !== "win32")( + "rejects a reply token embedded in extra prose (#8942)", + () => { + for (const reply of [ + "The answer is PONG, with extra prose.", + "The answer is \u001b[31mPONG\u001b[0m, with extra prose.", + ]) { + const result = runLaunchTurnFixture(0, reply, true); + + expect(result.signal, result.stderr).toBeNull(); + expect(result.status).toBe(1); + expect(result.stderr).toContain("launch did not produce the expected agent reply"); + expect(result.stdout).not.toContain("NEMOCLAW_LAUNCH_TURN_OK"); + } + }, +); + it.runIf(process.platform !== "win32")( "reports a nonzero TUI exit after recording a successful reply (#8584)", () => { diff --git a/test/helpers/launch-readiness-fixture.ts b/test/helpers/launch-readiness-fixture.ts new file mode 100644 index 00000000000..c8077c41b2b --- /dev/null +++ b/test/helpers/launch-readiness-fixture.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_GATEWAY_PORT } from "../../src/lib/core/ports"; +import { resolveGatewayName } from "../../src/lib/onboard/gateway-binding"; +import { fingerprintSandboxRecreateValue } from "../../src/lib/onboard/sandbox-recreate-transaction"; + +export const LAUNCH_READINESS_FIXTURE_POLICY = `version: 1 +network_policies: + fixture_api: + name: Fixture API + endpoints: + - host: example.com + port: 443 + binaries: + - path: /usr/bin/curl +`; + +export function launchReadinessRegistryFixture(sandboxId = "abc") { + return { + openshellDriver: "docker", + openshellVersion: "0.0.16", + gatewayName: resolveGatewayName(DEFAULT_GATEWAY_PORT), + gatewayPort: DEFAULT_GATEWAY_PORT, + lifecycleGeneration: "launch-readiness-fixture-generation", + lifecycleLiveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), + } as const; +} diff --git a/test/helpers/vitest-temp-root.ts b/test/helpers/vitest-temp-root.ts index 69c17963c14..cd52e88b53d 100644 --- a/test/helpers/vitest-temp-root.ts +++ b/test/helpers/vitest-temp-root.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +11,68 @@ const TEMP_ENV_KEYS = ["TMPDIR", "TMP", "TEMP"] as const; type TempEnvKey = (typeof TEMP_ENV_KEYS)[number]; +type RuntimeAuthorityInstaller = ( + command: string, + args: string[], + options: { stdio: "inherit" }, +) => void; + +interface GitHubHostedRuntimeAuthorityOptions { + platform?: NodeJS.Platform; + githubActions?: string; + runnerEnvironment?: string; + runnerImageOs?: string; + uid?: number; + gid?: number; + install?: RuntimeAuthorityInstaller; +} + +export function prepareGitHubHostedRuntimeAuthority( + options: GitHubHostedRuntimeAuthorityOptions = {}, +): void { + const hostedRunner = + (options.runnerEnvironment ?? process.env.RUNNER_ENVIRONMENT) === "github-hosted" || + /^(?:ubuntu|macos|win)/i.test(options.runnerImageOs ?? process.env.ImageOS ?? ""); + if ( + (options.platform ?? process.platform) !== "linux" || + (options.githubActions ?? process.env.GITHUB_ACTIONS) !== "true" || + !hostedRunner + ) { + return; + } + const uid = options.uid ?? process.getuid?.(); + const gid = options.gid ?? process.getgid?.(); + if (!Number.isSafeInteger(uid) || !Number.isSafeInteger(gid)) { + throw new Error("GitHub-hosted launch-readiness tests require a numeric user identity"); + } + const runtimeRoot = `/run/user/${uid}`; + const productRoot = `${runtimeRoot}/nemoclaw`; + const authorityRoot = `${productRoot}/launch-readiness`; + const install = + options.install ?? + ((command: string, args: string[], execOptions: { stdio: "inherit" }): void => { + execFileSync(command, args, execOptions); + }); + install( + "sudo", + [ + "--non-interactive", + "install", + "-d", + "-m", + "0700", + "-o", + String(uid), + "-g", + String(gid), + runtimeRoot, + productRoot, + authorityRoot, + ], + { stdio: "inherit" }, + ); +} + function restoreTempEnv(previous: ReadonlyMap): void { for (const key of TEMP_ENV_KEYS) { const value = previous.get(key); @@ -71,4 +134,7 @@ export function setupVitestTempRoot(): () => void { }; } -export default setupVitestTempRoot; +export default function setupVitestEnvironment(): () => void { + prepareGitHubHostedRuntimeAuthority(); + return setupVitestTempRoot(); +} diff --git a/test/launch-readiness-forward-observation.test.ts b/test/launch-readiness-forward-observation.test.ts new file mode 100644 index 00000000000..a6c6548759b --- /dev/null +++ b/test/launch-readiness-forward-observation.test.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, expect, it, vi } from "vitest"; +import * as forwardHealth from "../src/lib/actions/sandbox/forward-health.ts"; +import { areSandboxLaunchForwardsHealthy } from "../src/lib/actions/sandbox/forward-recovery.ts"; +import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.ts"; +import * as agentRuntime from "../src/lib/agent/runtime.ts"; +import * as registry from "../src/lib/state/registry.ts"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function mockLaunchForwardObservation( + result: { status: number | null; output: string }, + reachable = true, + gatewayRuntime = true, +) { + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + runtime: { kind: gatewayRuntime ? "gateway" : "terminal" }, + forward_ports: [18790], + } as never); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(reachable); + return vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue(result); +} + +it("checks launch forwards through the sandbox's owning gateway without repair (#8942)", () => { + const capture = mockLaunchForwardObservation({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS +beta 127.0.0.1 18789 12345 running +beta 127.0.0.1 18790 12346 running`, + }); + + expect(areSandboxLaunchForwardsHealthy("beta")).toBe(true); + expect(capture).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledWith(["forward", "list", "--gateway", "nemoclaw"], { + ignoreError: true, + timeout: expect.any(Number), + }); +}); + +it("rejects a reachable listener when the owning forward row is missing (#8942)", () => { + mockLaunchForwardObservation({ + status: 0, + output: "SANDBOX BIND PORT PID STATUS", + }); + + expect(areSandboxLaunchForwardsHealthy("beta", "nemoclaw")).toBe(false); +}); + +it("returns unknown when the owner-scoped forward observation fails (#8942)", () => { + mockLaunchForwardObservation({ status: 1, output: "" }); + + expect(areSandboxLaunchForwardsHealthy("beta", "nemoclaw")).toBeNull(); +}); + +it("rejects an owning-gateway mismatch before the no-forward shortcut (#8942)", () => { + const capture = mockLaunchForwardObservation({ status: 0, output: "" }, true, false); + + expect(areSandboxLaunchForwardsHealthy("beta", "ambient-sibling")).toBe(false); + expect(capture).not.toHaveBeenCalled(); +}); diff --git a/test/recover-port-forward.test.ts b/test/recover-port-forward.test.ts index 2c4a84fbce0..465851172df 100644 --- a/test/recover-port-forward.test.ts +++ b/test/recover-port-forward.test.ts @@ -7,6 +7,10 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { + LAUNCH_READINESS_FIXTURE_POLICY, + launchReadinessRegistryFixture, +} from "./helpers/launch-readiness-fixture"; import { nonWslPlatformNodeOptions } from "./helpers/platform-override-node-options"; import { execTimeout, testTimeoutOptions } from "./helpers/timeouts"; @@ -126,6 +130,7 @@ function setupFixture(opts: { sandboxes: { [sandboxName]: { name: sandboxName, + ...launchReadinessRegistryFixture(), model: "nvidia/test-model", provider: "nvidia-prod", gpuEnabled: false, @@ -265,7 +270,8 @@ if (args[0] === "forward") { } if (args[0] === "policy" && args[1] === "get") { - process.exit(1); + process.stdout.write(${JSON.stringify(LAUNCH_READINESS_FIXTURE_POLICY)}); + process.exit(0); } if (args[0] === "inference" && args[1] === "get") { diff --git a/test/sandbox-connect-inference/auto-pair-approval.test.ts b/test/sandbox-connect-inference/auto-pair-approval.test.ts index a9642ee99c3..20d8413b8a6 100644 --- a/test/sandbox-connect-inference/auto-pair-approval.test.ts +++ b/test/sandbox-connect-inference/auto-pair-approval.test.ts @@ -19,6 +19,8 @@ import { setupFixture, } from "./helpers"; +const expectedProbeOnlyExitCode = process.platform === "darwin" ? 1 : 0; + function findApprovalExec(state: { sandboxExecCalls: string[][]; sandboxExecInputs: string[]; @@ -267,11 +269,11 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = }, "anthropic-prod", "claude-sonnet-4-20250514", - { gatewaySupervisorRecovery: true }, + { gatewaySupervisorRecovery: true, launchReadinessRegistry: true }, ); const result = runConnect(tmpDir, sandboxName, {}, ["--probe-only"]); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(expectedProbeOnlyExitCode); const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); const controlExec = findGatewayControlExec(state.dockerCalls as string[][]); @@ -300,11 +302,11 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = ); it( - "does not fail the recover path when the probe approval pass errors", + "keeps the platform probe outcome when the approval pass errors", testTimeoutOptions(20_000), () => { - // Best-effort: even when the in-sandbox approval exec exits non-zero, the - // probe-only flow must still succeed. + // Best-effort: an in-sandbox approval error does not change the platform + // result from readiness publication (success on Linux, unavailable on macOS). const { tmpDir, stateFile, sandboxName } = setupFixture( { name: "probe-approval-tol", @@ -315,13 +317,13 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = }, "anthropic-prod", "claude-sonnet-4-20250514", - { gatewaySupervisorRecovery: true }, + { gatewaySupervisorRecovery: true, launchReadinessRegistry: true }, ); const result = runConnect(tmpDir, sandboxName, { OPENSHELL_TEST_FAIL_APPROVAL_PASS: "1" }, [ "--probe-only", ]); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(expectedProbeOnlyExitCode); const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); const approvalExec = findApprovalExec(state); @@ -347,6 +349,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = }, "anthropic-prod", "claude-sonnet-4-20250514", + { launchReadinessRegistry: true }, ); const result = runConnect(tmpDir, sandboxName, { OPENSHELL_TEST_GATEWAY_DOWN: "1" }, [ @@ -381,11 +384,11 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = }, "anthropic-prod", "claude-sonnet-4-20250514", - { gatewaySupervisorRecovery: true }, + { gatewaySupervisorRecovery: true, launchReadinessRegistry: true }, ); const result = runConnect(tmpDir, sandboxName, {}, ["--probe-only"]); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(expectedProbeOnlyExitCode); const script = extractApprovalPassScript(stateFile, sandboxName); expect(script).toContain("approve_env = gateway_approval_env(os.environ)"); @@ -420,11 +423,11 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = }, "anthropic-prod", "claude-sonnet-4-20250514", - { gatewaySupervisorRecovery: true }, + { gatewaySupervisorRecovery: true, launchReadinessRegistry: true }, ); const result = runConnect(tmpDir, sandboxName, {}, ["--probe-only"]); - expect(result.status).toBe(0); + expect(result.status).toBe(expectedProbeOnlyExitCode); const script = extractApprovalPassScript(stateFile, sandboxName); // The rendered script interpolates the exported budget constants, tying diff --git a/test/sandbox-connect-inference/helpers.ts b/test/sandbox-connect-inference/helpers.ts index 899b18d7a7c..054dd5d784c 100644 --- a/test/sandbox-connect-inference/helpers.ts +++ b/test/sandbox-connect-inference/helpers.ts @@ -6,6 +6,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, expect } from "vitest"; +import { + LAUNCH_READINESS_FIXTURE_POLICY, + launchReadinessRegistryFixture, +} from "../helpers/launch-readiness-fixture"; import { nonWslPlatformNodeOptions } from "../helpers/platform-override-node-options"; import { execTimeout } from "../helpers/timeouts"; @@ -36,6 +40,7 @@ export type SetupFixtureOptions = { inferenceSetStatus?: number; writeOllamaProxyState?: boolean; gatewaySupervisorRecovery?: boolean; + launchReadinessRegistry?: boolean; }; const fixtureForwardListeners = new Map(); @@ -133,7 +138,12 @@ function writeRegistryState( path.join(registryDir, "sandboxes.json"), JSON.stringify({ defaultSandbox: sandboxName, - sandboxes: { [sandboxName]: sandboxEntry }, + sandboxes: { + [sandboxName]: { + ...(options.launchReadinessRegistry ? launchReadinessRegistryFixture() : {}), + ...sandboxEntry, + }, + }, }), { mode: 0o600 }, ); @@ -171,7 +181,7 @@ function initStateFile(stateFile: string, options: SetupFixtureOptions) { curlCalls: [], curlEnvs: [], inferenceProbeExitStatuses: options.inferenceProbeExitStatuses ?? [], - inferenceProbeResponses: options.inferenceProbeResponses ?? ["OK 200"], + inferenceProbeResponses: options.inferenceProbeResponses ?? ["OK 200", "OK 200"], inferenceGetCalls: [], inferenceSetCalls: [], sandboxConnectCalls: [], @@ -214,7 +224,11 @@ if (args[0] === "gateway" && args[1] === "info") { process.exit(0); } -if (args[0] === "sandbox" && args[1] === "get" && args[2] === ${JSON.stringify(sandboxName)}) { +if ( + args[0] === "sandbox" && + args[1] === "get" && + args[args.length - 1] === ${JSON.stringify(sandboxName)} +) { process.stdout.write("Sandbox:\\n\\n \\x1b[2mId:\\x1b[0m abc\\n Name: ${sandboxName}\\n Phase: Ready\\n"); process.exit(0); } @@ -283,6 +297,11 @@ if (args[0] === "inference" && args[1] === "get") { process.exit(0); } +if (args[0] === "policy" && args[1] === "get") { + process.stdout.write(${JSON.stringify(LAUNCH_READINESS_FIXTURE_POLICY)}); + process.exit(0); +} + if (args[0] === "inference" && args[1] === "set") { state.inferenceSetCalls.push(args.slice(2)); fs.writeFileSync(stateFile, JSON.stringify(state)); diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index c9bf4807210..1a049d8ad80 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -13,6 +13,8 @@ import type { SandboxEntry } from "../../src/lib/state/registry"; type ConnectSandbox = typeof import("../../src/lib/actions/sandbox/connect")["connectSandbox"]; type GatewayRouteMutationLock = typeof import("../../src/lib/inference/gateway-route-mutation-lock")["withGatewayRouteMutationLock"]; +type LaunchReadinessPublicationResult = + import("../../src/lib/actions/sandbox/launch-readiness").LaunchReadinessPublicationResult; export const requireDist = createRequire(import.meta.url); export const connectModulePath = "../../src/lib/actions/sandbox/connect.js"; @@ -32,6 +34,9 @@ export type ConnectHarness = { ensureLiveSandboxSpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; + inspectLaunchReadinessSpy: MockInstance; + launchReadinessMutationGateSpy: MockInstance; + publishLaunchReadinessSpy: MockInstance; preflightVllmSpy: MockInstance; probeLocalProviderHealthSpy: MockInstance; probeOllamaAuthProxyHealthSpy: MockInstance; @@ -50,6 +55,7 @@ export type ConnectHarness = { export type ConnectHarnessOptions = { agentName?: string; inferenceGetOutput?: string; + isWsl?: boolean; inferenceProbeResponses?: Array< string | { status?: number | null; output?: string | null; stderr?: string | null } >; @@ -77,6 +83,19 @@ export type ConnectHarnessOptions = { spawnStatus?: number | null; sttyThrows?: boolean; withGatewayRouteMutationLock?: GatewayRouteMutationLock; + readinessDecision?: + | { kind: "accepted"; category: "accepted"; agent: unknown; sb: SandboxEntry } + | { + kind: "fallback"; + category: string; + fence: { epochId: string } | null; + gatewayName: string | null; + gatewayPort: number | null; + fenceFailed: boolean; + recoveryBlocked: boolean; + authorityUnsupported?: true; + }; + readinessPublicationResult?: LaunchReadinessPublicationResult; }; function throwSttyFailure(): never { @@ -119,6 +138,7 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne ); const localInference = requireDist("../../src/lib/inference/local.js"); const ollamaProxy = requireDist("../../src/lib/inference/ollama/proxy.js"); + const platform = requireDist("../../src/lib/platform.js"); const gatewayRouteMutationLock = requireDist( "../../src/lib/inference/gateway-route-mutation-lock.js", ); @@ -127,6 +147,33 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne const registry = requireDist("../../src/lib/state/registry.js"); const sandboxSession = requireDist("../../src/lib/state/sandbox-session.js"); const vmDnsMonkeypatch = requireDist("../../src/lib/actions/sandbox/vm-dns-monkeypatch.js"); + const launchReadiness = requireDist("../../src/lib/actions/sandbox/launch-readiness.js"); + + const inspectLaunchReadinessSpy = vi + .spyOn(launchReadiness, "inspectLaunchReadiness") + .mockResolvedValue( + options.readinessDecision ?? { + kind: "fallback", + category: "missing", + fence: { epochId: "a".repeat(64) }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + fenceFailed: false, + recoveryBlocked: false, + }, + ); + const publishLaunchReadinessSpy = vi + .spyOn(launchReadiness, "publishLaunchReadiness") + .mockResolvedValue(options.readinessPublicationResult ?? { kind: "published" }); + const launchReadinessMutationGateSpy = vi + .spyOn(launchReadiness, "withLaunchReadinessMutationGate") + .mockImplementation((async (...args: unknown[]) => { + const operation = args[1] as () => unknown; + return { kind: "entered", value: await operation() }; + }) as never); + if (typeof options.isWsl === "boolean") { + vi.spyOn(platform, "isWsl").mockReturnValue(options.isWsl); + } const preflightVllmSpy = vi .spyOn(connectVllmPreflight, "preflightVllmModelEnvOrExit") @@ -278,6 +325,9 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne ensureLiveSandboxSpy, errorSpy, logSpy, + inspectLaunchReadinessSpy, + launchReadinessMutationGateSpy, + publishLaunchReadinessSpy, preflightVllmSpy, probeLocalProviderHealthSpy, probeOllamaAuthProxyHealthSpy, diff --git a/test/vitest-temp-root.test.ts b/test/vitest-temp-root.test.ts index eb781c21a8c..01deb8be61f 100644 --- a/test/vitest-temp-root.test.ts +++ b/test/vitest-temp-root.test.ts @@ -9,7 +9,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import pluginVitestConfig from "../nemoclaw/vitest.config"; import rootVitestConfig from "../vitest.config"; -import { setupVitestTempRoot } from "./helpers/vitest-temp-root"; +import { + prepareGitHubHostedRuntimeAuthority, + setupVitestTempRoot, +} from "./helpers/vitest-temp-root"; const TEMP_ENV_KEYS = ["TMPDIR", "TMP", "TEMP"] as const; const ROOT_SETUP = "test/helpers/vitest-temp-root.ts"; @@ -41,6 +44,71 @@ afterEach(() => { }); describe("Vitest temp root", () => { + it("reapplies the required Linux runtime-authority ownership and mode (#8942)", () => { + const install = vi.fn(); + const options = { + platform: "linux" as const, + githubActions: "true", + runnerEnvironment: "github-hosted", + uid: 1001, + gid: 1002, + install, + }; + + prepareGitHubHostedRuntimeAuthority(options); + prepareGitHubHostedRuntimeAuthority(options); + + expect(install).toHaveBeenCalledTimes(2); + expect(install).toHaveBeenLastCalledWith( + "sudo", + [ + "--non-interactive", + "install", + "-d", + "-m", + "0700", + "-o", + "1001", + "-g", + "1002", + "/run/user/1001", + "/run/user/1001/nemoclaw", + "/run/user/1001/nemoclaw/launch-readiness", + ], + { stdio: "inherit" }, + ); + }); + + it("does not prepare runtime authority on macOS (#8942)", () => { + const install = vi.fn(); + + prepareGitHubHostedRuntimeAuthority({ + platform: "darwin", + githubActions: "true", + runnerEnvironment: "github-hosted", + uid: 1001, + gid: 1002, + install, + }); + + expect(install).not.toHaveBeenCalled(); + }); + + it("recognizes the GitHub-hosted image marker exported to test processes (#8942)", () => { + const install = vi.fn(); + + prepareGitHubHostedRuntimeAuthority({ + platform: "linux", + githubActions: "true", + runnerImageOs: "ubuntu24", + uid: 1001, + gid: 1002, + install, + }); + + expect(install).toHaveBeenCalledOnce(); + }); + it("redirects the selected project into one private temp root", () => { const root = process.env.TMPDIR as string;