diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e3cbf06d86..84ed59da5a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2893,6 +2893,11 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" OPENSHELL_GATEWAY: "nemoclaw" + # Selects the shared E2E inference adapter's internal-nvidia mode + # (#5745) so this real-latency measurement always runs against a real + # hosted endpoint; the adapter defaults to hermetic `mock` otherwise, + # which the test now explicitly refuses to measure latency against. + NEMOCLAW_E2E_INFERENCE_MODE: "internal-nvidia" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_PROVIDER: custom NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index 5217f61f18..8410da56b2 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -77,7 +77,7 @@ const MODEL_PROBE_TIMEOUT_MS = 30_000; const PUBLIC_NVIDIA_ALLOWED_HOSTS = ["integrate.api.nvidia.com"] as const; const SANDBOX_HOST_ALIAS = "host.openshell.internal"; -function normalizeMode(env: NodeJS.ProcessEnv): E2EInferenceMode { +export function normalizeMode(env: NodeJS.ProcessEnv): E2EInferenceMode { const raw = env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase(); if (!raw) return "mock"; if (raw === "mock" || raw === "internal-nvidia" || raw === "public-nvidia") return raw; diff --git a/test/e2e/live/agent-turn-latency-helpers.ts b/test/e2e/live/agent-turn-latency-helpers.ts index 91503a70b5..e01ab3464d 100644 --- a/test/e2e/live/agent-turn-latency-helpers.ts +++ b/test/e2e/live/agent-turn-latency-helpers.ts @@ -12,11 +12,20 @@ import { validateSandboxName, } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import type { E2EInferenceAdapter } from "../fixtures/inference-adapter.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; +// The injected E2E inference adapter (#5745) is the single source of the +// model, provider, expected route, and credential this suite exercises; +// this file must not rederive them from ad hoc NEMOCLAW_* env inspection. +export type AgentTurnInference = Pick< + E2EInferenceAdapter, + "env" | "expectedRouteProvider" | "model" | "mode" | "provider" | "redactionValues" +>; + export { REPO_ROOT }; export const CLI = CLI_ENTRYPOINT; @@ -26,19 +35,6 @@ export const HERMES_SANDBOX = process.env.NEMOCLAW_HERMES_TURN_LATENCY_SANDBOX_NAME ?? "e2e-hm-turn-lat"; validateSandboxName(OPENCLAW_SANDBOX); validateSandboxName(HERMES_SANDBOX); -const DEFAULT_NVIDIA_MODEL = "nvidia/nemotron-3-super-120b-a12b"; -const DEFAULT_COMPAT_MODEL = "nvidia/nvidia/nemotron-3-ultra"; -const USE_COMPATIBLE_HOSTED = process.env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE === "1"; -export const MODEL = - process.env.NEMOCLAW_TURN_LATENCY_MODEL ?? - process.env.NEMOCLAW_MODEL ?? - process.env.NEMOCLAW_COMPAT_MODEL ?? - (USE_COMPATIBLE_HOSTED ? DEFAULT_COMPAT_MODEL : DEFAULT_NVIDIA_MODEL); -const PROVIDER = - process.env.NEMOCLAW_TURN_LATENCY_PROVIDER ?? (USE_COMPATIBLE_HOSTED ? "custom" : "build"); -export const EXPECTED_ROUTE_PROVIDER = - process.env.NEMOCLAW_TURN_LATENCY_ROUTE_PROVIDER ?? - (PROVIDER === "custom" ? "compatible-endpoint" : "nvidia-prod"); export const MAX_TURN_SECONDS = positiveInt(process.env.NEMOCLAW_TURN_LATENCY_MAX_SECONDS, 300); const INSTALL_ATTEMPTS = positiveInt(process.env.NEMOCLAW_TURN_LATENCY_INSTALL_ATTEMPTS, 2); const INSTALL_TIMEOUT_MS = 30 * 60_000; @@ -52,30 +48,25 @@ function positiveInt(value: string | undefined, fallback: number): number { export function env( sandboxName: string, agent: "openclaw" | "hermes", - apiKey?: string, + inference: AgentTurnInference, + includeCredential = false, ): NodeJS.ProcessEnv { - const out: NodeJS.ProcessEnv = { + const base: NodeJS.ProcessEnv = { ...buildAvailabilityProbeEnv(), NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_MODEL: MODEL, NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_FRESH: "1", - NEMOCLAW_PROVIDER: PROVIDER, NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_SANDBOX_NAME: sandboxName, OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", }; - agent === "hermes" && (out.NEMOCLAW_AGENT = "hermes"); - apiKey && Object.assign(out, { NVIDIA_INFERENCE_API_KEY: apiKey }); - PROVIDER === "custom" && - Object.assign(out, { - COMPATIBLE_API_KEY: apiKey, - NEMOCLAW_COMPAT_MODEL: MODEL, - NEMOCLAW_ENDPOINT_URL: - process.env.NEMOCLAW_ENDPOINT_URL ?? "https://inference-api.nvidia.com/v1", - NEMOCLAW_PREFERRED_API: process.env.NEMOCLAW_PREFERRED_API ?? "openai-completions", - }); - return out; + agent === "hermes" && (base.NEMOCLAW_AGENT = "hermes"); + if (!includeCredential) { + base.NEMOCLAW_PROVIDER = inference.provider; + base.NEMOCLAW_MODEL = inference.model; + return base; + } + return inference.env(base); } export async function bestEffortPreclean( @@ -347,7 +338,7 @@ export async function installSandbox( host: HostCliClient, sandboxName: string, agent: "openclaw" | "hermes", - apiKey: string, + inference: AgentTurnInference, cleanupBeforeRetry?: () => Promise, progress?: AgentTurnProgress, ): Promise { @@ -366,9 +357,9 @@ export async function installSandbox( { artifactName: `${agent}-install-attempt-${attempt}`, cwd: REPO_ROOT, - env: env(sandboxName, agent, apiKey), + env: env(sandboxName, agent, inference, true), onOutput: progress?.onOutput, - redactionValues: [apiKey], + redactionValues: inference.redactionValues(), timeoutMs: INSTALL_TIMEOUT_MS, }, ); @@ -419,6 +410,7 @@ export async function installSandbox( export async function cleanupTurnSandboxes( host: HostCliClient, sandbox: SandboxClient, + inference: AgentTurnInference, progress?: AgentTurnProgress, ): Promise { for (const [name, agent] of [ @@ -427,7 +419,7 @@ export async function cleanupTurnSandboxes( ] as const) { await runBestEffortCleanupStep( `destroy ${agent} sandbox`, - () => cleanupTurnSandbox(host, name, agent, progress), + () => cleanupTurnSandbox(host, name, agent, inference, progress), progress, ); await runBestEffortCleanupStep( @@ -435,7 +427,7 @@ export async function cleanupTurnSandboxes( () => sandbox.openshell(["sandbox", "delete", name], { artifactName: `cleanup-${agent}-delete`, - env: env(name, agent), + env: env(name, agent, inference), onOutput: progress?.onOutput, timeoutMs: 60_000, }), @@ -470,11 +462,12 @@ export async function cleanupTurnSandbox( host: HostCliClient, name: string, agent: "openclaw" | "hermes", + inference: AgentTurnInference, progress?: Pick, ): Promise { const result = await host.command("node", [CLI, name, "destroy", "--yes"], { artifactName: `cleanup-${agent}-destroy`, - env: env(name, agent), + env: env(name, agent, inference), onOutput: progress?.onOutput, timeoutMs: 120_000, }); @@ -492,12 +485,13 @@ export async function route( sandbox: SandboxClient, sandboxName: string, agent: "openclaw" | "hermes", + inference: AgentTurnInference, artifactName: string, progress?: Pick, ): Promise { return await sandbox.openshell(["inference", "get", "-g", "nemoclaw"], { artifactName, - env: env(sandboxName, agent), + env: env(sandboxName, agent, inference), onOutput: progress?.onOutput, timeoutMs: 30_000, }); @@ -505,7 +499,7 @@ export async function route( export async function openclawTurn( sandbox: SandboxClient, - apiKey: string, + inference: AgentTurnInference, progress?: Pick, ): Promise<{ result: ShellProbeResult; elapsedMs: number }> { const started = process.hrtime.bigint(); @@ -516,9 +510,9 @@ export async function openclawTurn( ), { artifactName: "openclaw-agent-turn", - env: env(OPENCLAW_SANDBOX, "openclaw"), + env: env(OPENCLAW_SANDBOX, "openclaw", inference), onOutput: progress?.onOutput, - redactionValues: [apiKey], + redactionValues: inference.redactionValues(), timeoutMs: (MAX_TURN_SECONDS + 30) * 1000, }, ); @@ -527,6 +521,7 @@ export async function openclawTurn( export async function waitHermesHealth( sandbox: SandboxClient, + inference: AgentTurnInference, progress?: Pick, ): Promise { return await sandbox.execShell( @@ -536,7 +531,7 @@ export async function waitHermesHealth( ), { artifactName: "hermes-health", - env: env(HERMES_SANDBOX, "hermes"), + env: env(HERMES_SANDBOX, "hermes", inference), onOutput: progress?.onOutput, timeoutMs: 90_000, }, diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index f7e797ae75..8f3652e368 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -8,6 +8,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { normalizeMode } from "../fixtures/inference-adapter.ts"; import { assertHermesConfig, assertNoOpenClawTransportErrors, @@ -16,14 +17,12 @@ import { chatContent, cleanupTurnSandbox, cleanupTurnSandboxes, - EXPECTED_ROUTE_PROVIDER, env, extractOpenClawAgentText, HERMES_SANDBOX, hermesTurnCommand, installSandbox, MAX_TURN_SECONDS, - MODEL, OPENCLAW_SANDBOX, openclawConfigCommand, openclawTurn, @@ -34,194 +33,220 @@ import { const TIMEOUT_MS = 90 * 60_000; -test("OpenClaw and Hermes complete real hosted inference turns within the latency cap", { - timeout: TIMEOUT_MS, - meta: { - e2ePhases: [ - "prepare clean inference hosts", - "install OpenClaw sandbox", - "validate OpenClaw inference route", - "run OpenClaw hosted inference turn", - "replace OpenClaw with Hermes sandbox", - "validate Hermes inference route", - "run Hermes hosted inference turn", - "record hosted inference timing evidence", - ], +// A real latency measurement needs a real hosted endpoint; the shared +// adapter's hermetic `mock` mode would just measure a loopback round trip +// and report a meaningless number. Select `internal-nvidia` or +// `public-nvidia` via NEMOCLAW_E2E_INFERENCE_MODE for this target. Resolved +// at module scope (matching issue-4434's runIssue4434LiveTest pattern) so +// the skip is a test-definition boundary, not a conditional test body. +const runAgentTurnLatencyTest = test.skipIf(normalizeMode(process.env) === "mock"); + +runAgentTurnLatencyTest( + "OpenClaw and Hermes complete real hosted inference turns within the latency cap", + { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "prepare clean inference hosts", + "install OpenClaw sandbox", + "validate OpenClaw inference route", + "run OpenClaw hosted inference turn", + "replace OpenClaw with Hermes sandbox", + "validate Hermes inference route", + "run Hermes hosted inference turn", + "record hosted inference timing evidence", + ], + }, }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const results: Record = { model: MODEL, maxTurnSeconds: MAX_TURN_SECONDS }; - await artifacts.target.declare({ - id: "agent-turn-latency", - boundary: "two real sandboxes + hosted inference + OpenClaw agent turn + Hermes API turn", - openclawSandbox: OPENCLAW_SANDBOX, - hermesSandbox: HERMES_SANDBOX, - }); - cleanup.trackDisposable("remove gateway nemoclaw", async () => { - await host.cleanupGatewayRegistration("nemoclaw", { - artifactName: "cleanup-gateway-destroy-turn-latency", - env: buildAvailabilityProbeEnv(), - onOutput: progress.onOutput, - timeoutMs: 60_000, + async ({ artifacts, cleanup, host, inference, progress, sandbox }) => { + const results: Record = { + model: inference.model, + maxTurnSeconds: MAX_TURN_SECONDS, + }; + await artifacts.target.declare({ + id: "agent-turn-latency", + boundary: "two real sandboxes + hosted inference + OpenClaw agent turn + Hermes API turn", + openclawSandbox: OPENCLAW_SANDBOX, + hermesSandbox: HERMES_SANDBOX, }); - }); - cleanup.trackDisposable("stop forward 8642", async () => { - await host.cleanupForward(8642, { - artifactName: "cleanup-forward-stop-hermes-api", - env: buildAvailabilityProbeEnv(), - onOutput: progress.onOutput, - timeoutMs: 30_000, + cleanup.trackDisposable("remove gateway nemoclaw", async () => { + await host.cleanupGatewayRegistration("nemoclaw", { + artifactName: "cleanup-gateway-destroy-turn-latency", + env: buildAvailabilityProbeEnv(), + onOutput: progress.onOutput, + timeoutMs: 60_000, + }); }); - }); - cleanup.trackDisposable("delete Hermes OpenShell sandbox", async () => { - await sandbox.cleanupSandbox(HERMES_SANDBOX, { - artifactName: "cleanup-hermes-delete", - env: env(HERMES_SANDBOX, "hermes"), - onOutput: progress.onOutput, - timeoutMs: 60_000, + cleanup.trackDisposable("stop forward 8642", async () => { + await host.cleanupForward(8642, { + artifactName: "cleanup-forward-stop-hermes-api", + env: buildAvailabilityProbeEnv(), + onOutput: progress.onOutput, + timeoutMs: 30_000, + }); }); - }); - cleanup.trackDisposable("destroy Hermes sandbox", async () => { - await cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes", progress); - }); - cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", async () => { - await sandbox.cleanupSandbox(OPENCLAW_SANDBOX, { - artifactName: "cleanup-openclaw-delete", - env: env(OPENCLAW_SANDBOX, "openclaw"), - onOutput: progress.onOutput, - timeoutMs: 60_000, + cleanup.trackDisposable("delete Hermes OpenShell sandbox", async () => { + await sandbox.cleanupSandbox(HERMES_SANDBOX, { + artifactName: "cleanup-hermes-delete", + env: env(HERMES_SANDBOX, "hermes", inference), + onOutput: progress.onOutput, + timeoutMs: 60_000, + }); + }); + cleanup.trackDisposable("destroy Hermes sandbox", async () => { + await cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes", inference, progress); + }); + cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", async () => { + await sandbox.cleanupSandbox(OPENCLAW_SANDBOX, { + artifactName: "cleanup-openclaw-delete", + env: env(OPENCLAW_SANDBOX, "openclaw", inference), + onOutput: progress.onOutput, + timeoutMs: 60_000, + }); + }); + cleanup.trackDisposable("destroy OpenClaw sandbox", async () => { + await cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw", inference, progress); }); - }); - cleanup.trackDisposable("destroy OpenClaw sandbox", async () => { - await cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw", progress); - }); - - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - onOutput: progress.onOutput, - timeoutMs: 30_000, - }); - expect(docker.exitCode, resultText(docker)).toBe(0); - const cleanBeforeRetry = () => cleanupTurnSandboxes(host, sandbox, progress); - await cleanupTurnSandboxes(host, sandbox, progress); - progress.phase("install OpenClaw sandbox"); - const openclawInstall = await installSandbox( - host, - OPENCLAW_SANDBOX, - "openclaw", - apiKey, - cleanBeforeRetry, - progress, - ); - expect(openclawInstall.exitCode, resultText(openclawInstall)).toBe(0); - progress.phase("validate OpenClaw inference route"); - const openclawRoute = await route( - sandbox, - OPENCLAW_SANDBOX, - "openclaw", - "openclaw-route", - progress, - ); - expect(openclawRoute.exitCode, resultText(openclawRoute)).toBe(0); - expect(resultText(openclawRoute)).toContain(EXPECTED_ROUTE_PROVIDER); - expect(resultText(openclawRoute)).toContain(MODEL); - const openclawConfig = await sandbox.execShell( - OPENCLAW_SANDBOX, - trustedSandboxShellScript(openclawConfigCommand()), - { - artifactName: "openclaw-config", - env: env(OPENCLAW_SANDBOX, "openclaw"), + const docker = await host.command("docker", ["info"], { + artifactName: "docker-info", + env: buildAvailabilityProbeEnv(), onOutput: progress.onOutput, - redactionValues: [apiKey], timeoutMs: 30_000, - }, - ); - expect(openclawConfig.exitCode, resultText(openclawConfig)).toBe(0); - assertOpenClawConfig(openclawConfig.stdout, MODEL); + }); + expect(docker.exitCode, resultText(docker)).toBe(0); - progress.phase("run OpenClaw hosted inference turn"); - const openclaw = await openclawTurn(sandbox, apiKey, progress); - expect(openclaw.result.exitCode, resultText(openclaw.result)).toBe(0); - assertNoOpenClawTransportErrors(resultText(openclaw.result)); - expect( - containsInteger42Answer(extractOpenClawAgentText(openclaw.result.stdout)), - resultText(openclaw.result), - ).toBe(true); - expect(openclaw.elapsedMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); - results.openclaw = { elapsedMs: openclaw.elapsedMs }; + const cleanBeforeRetry = () => cleanupTurnSandboxes(host, sandbox, inference, progress); + await cleanupTurnSandboxes(host, sandbox, inference, progress); + progress.phase("install OpenClaw sandbox"); + const openclawInstall = await installSandbox( + host, + OPENCLAW_SANDBOX, + "openclaw", + inference, + cleanBeforeRetry, + progress, + ); + expect(openclawInstall.exitCode, resultText(openclawInstall)).toBe(0); + progress.phase("validate OpenClaw inference route"); + const openclawRoute = await route( + sandbox, + OPENCLAW_SANDBOX, + "openclaw", + inference, + "openclaw-route", + progress, + ); + expect(openclawRoute.exitCode, resultText(openclawRoute)).toBe(0); + expect(resultText(openclawRoute)).toContain(inference.expectedRouteProvider); + expect(resultText(openclawRoute)).toContain(inference.model); + const openclawConfig = await sandbox.execShell( + OPENCLAW_SANDBOX, + trustedSandboxShellScript(openclawConfigCommand()), + { + artifactName: "openclaw-config", + env: env(OPENCLAW_SANDBOX, "openclaw", inference), + onOutput: progress.onOutput, + redactionValues: inference.redactionValues(), + timeoutMs: 30_000, + }, + ); + expect(openclawConfig.exitCode, resultText(openclawConfig)).toBe(0); + assertOpenClawConfig(openclawConfig.stdout, inference.model); - progress.phase("replace OpenClaw with Hermes sandbox"); - await host.command("node", [CLI, OPENCLAW_SANDBOX, "destroy", "--yes"], { - artifactName: "destroy-openclaw-before-hermes", - env: env(OPENCLAW_SANDBOX, "openclaw"), - onOutput: progress.onOutput, - timeoutMs: 120_000, - }); + progress.phase("run OpenClaw hosted inference turn"); + const openclaw = await openclawTurn(sandbox, inference, progress); + expect(openclaw.result.exitCode, resultText(openclaw.result)).toBe(0); + assertNoOpenClawTransportErrors(resultText(openclaw.result)); + expect( + containsInteger42Answer(extractOpenClawAgentText(openclaw.result.stdout)), + resultText(openclaw.result), + ).toBe(true); + expect(openclaw.elapsedMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); + results.openclaw = { elapsedMs: openclaw.elapsedMs }; - const hermesInstall = await installSandbox( - host, - HERMES_SANDBOX, - "hermes", - apiKey, - cleanBeforeRetry, - progress, - ); - expect(hermesInstall.exitCode, resultText(hermesInstall)).toBe(0); - progress.phase("validate Hermes inference route"); - const hermesRoute = await route(sandbox, HERMES_SANDBOX, "hermes", "hermes-route", progress); - expect(hermesRoute.exitCode, resultText(hermesRoute)).toBe(0); - expect(resultText(hermesRoute)).toContain(EXPECTED_ROUTE_PROVIDER); - expect(resultText(hermesRoute)).toContain(MODEL); - const hermesHealth = await waitHermesHealth(sandbox, progress); - expect(hermesHealth.exitCode, resultText(hermesHealth)).toBe(0); - const hermesConfig = await sandbox.exec(HERMES_SANDBOX, ["cat", "/sandbox/.hermes/config.yaml"], { - artifactName: "hermes-config", - env: env(HERMES_SANDBOX, "hermes"), - onOutput: progress.onOutput, - redactionValues: [apiKey], - timeoutMs: 30_000, - }); - expect(hermesConfig.exitCode, resultText(hermesConfig)).toBe(0); - assertHermesConfig(hermesConfig.stdout, MODEL); + progress.phase("replace OpenClaw with Hermes sandbox"); + await host.command("node", [CLI, OPENCLAW_SANDBOX, "destroy", "--yes"], { + artifactName: "destroy-openclaw-before-hermes", + env: env(OPENCLAW_SANDBOX, "openclaw", inference), + onOutput: progress.onOutput, + timeoutMs: 120_000, + }); - const payload = JSON.stringify({ - model: MODEL, - messages: [ + const hermesInstall = await installSandbox( + host, + HERMES_SANDBOX, + "hermes", + inference, + cleanBeforeRetry, + progress, + ); + expect(hermesInstall.exitCode, resultText(hermesInstall)).toBe(0); + progress.phase("validate Hermes inference route"); + const hermesRoute = await route( + sandbox, + HERMES_SANDBOX, + "hermes", + inference, + "hermes-route", + progress, + ); + expect(hermesRoute.exitCode, resultText(hermesRoute)).toBe(0); + expect(resultText(hermesRoute)).toContain(inference.expectedRouteProvider); + expect(resultText(hermesRoute)).toContain(inference.model); + const hermesHealth = await waitHermesHealth(sandbox, inference, progress); + expect(hermesHealth.exitCode, resultText(hermesHealth)).toBe(0); + const hermesConfig = await sandbox.exec( + HERMES_SANDBOX, + ["cat", "/sandbox/.hermes/config.yaml"], { - role: "user", - content: "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + artifactName: "hermes-config", + env: env(HERMES_SANDBOX, "hermes", inference), + onOutput: progress.onOutput, + redactionValues: inference.redactionValues(), + timeoutMs: 30_000, }, - ], - max_tokens: 64, - }); - progress.phase("run Hermes hosted inference turn"); - const hermesStarted = process.hrtime.bigint(); - const hermesTurn = await sandbox.execShell( - HERMES_SANDBOX, - trustedSandboxShellScript(hermesTurnCommand(payload)), - { - artifactName: "hermes-api-turn", - env: env(HERMES_SANDBOX, "hermes"), - onOutput: progress.onOutput, - redactionValues: [apiKey], - timeoutMs: (MAX_TURN_SECONDS + 30) * 1000, - }, - ); - const hermesMs = Number((process.hrtime.bigint() - hermesStarted) / 1_000_000n); - expect(hermesTurn.exitCode, resultText(hermesTurn)).toBe(0); - const hermesResponse = responseBodyAndStatus(hermesTurn.stdout); - expect(hermesResponse.status, resultText(hermesTurn)).toBe("200"); - expect(containsInteger42Answer(chatContent(hermesResponse.body)), resultText(hermesTurn)).toBe( - true, - ); - expect(hermesMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); - results.hermes = { elapsedMs: hermesMs }; - await artifacts.writeJson("turn-latency-results.json", results); - progress.phase("record hosted inference timing evidence"); - fs.writeFileSync( - artifacts.pathFor("agent-turn-latency-results-legacy-path.json"), - `${JSON.stringify(results, null, 2)}\n`, - ); -}); + ); + expect(hermesConfig.exitCode, resultText(hermesConfig)).toBe(0); + assertHermesConfig(hermesConfig.stdout, inference.model); + + const payload = JSON.stringify({ + model: inference.model, + messages: [ + { + role: "user", + content: "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + }, + ], + max_tokens: 64, + }); + progress.phase("run Hermes hosted inference turn"); + const hermesStarted = process.hrtime.bigint(); + const hermesTurn = await sandbox.execShell( + HERMES_SANDBOX, + trustedSandboxShellScript(hermesTurnCommand(payload)), + { + artifactName: "hermes-api-turn", + env: env(HERMES_SANDBOX, "hermes", inference), + onOutput: progress.onOutput, + redactionValues: inference.redactionValues(), + timeoutMs: (MAX_TURN_SECONDS + 30) * 1000, + }, + ); + const hermesMs = Number((process.hrtime.bigint() - hermesStarted) / 1_000_000n); + expect(hermesTurn.exitCode, resultText(hermesTurn)).toBe(0); + const hermesResponse = responseBodyAndStatus(hermesTurn.stdout); + expect(hermesResponse.status, resultText(hermesTurn)).toBe("200"); + expect(containsInteger42Answer(chatContent(hermesResponse.body)), resultText(hermesTurn)).toBe( + true, + ); + expect(hermesMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); + results.hermes = { elapsedMs: hermesMs }; + await artifacts.writeJson("turn-latency-results.json", results); + progress.phase("record hosted inference timing evidence"); + fs.writeFileSync( + artifacts.pathFor("agent-turn-latency-results-legacy-path.json"), + `${JSON.stringify(results, null, 2)}\n`, + ); + }, +); diff --git a/test/e2e/support/agent-turn-latency-progress.test.ts b/test/e2e/support/agent-turn-latency-progress.test.ts index 35a73ea3da..d8647723ca 100644 --- a/test/e2e/support/agent-turn-latency-progress.test.ts +++ b/test/e2e/support/agent-turn-latency-progress.test.ts @@ -11,12 +11,24 @@ import { validateE2EPhasePlan, } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import type { AgentTurnInference } from "../live/agent-turn-latency-helpers.ts"; import { bestEffortPreclean, cleanupTurnSandboxes, installSandbox, } from "../live/agent-turn-latency-helpers.ts"; +function fakeInference(apiKey = "secret-api-key"): AgentTurnInference { + return { + mode: "internal-nvidia", + model: "test-model", + provider: "custom", + expectedRouteProvider: "compatible-endpoint", + env: (extra = {}) => ({ ...extra, COMPATIBLE_API_KEY: apiKey }), + redactionValues: () => [apiKey], + }; +} + function progressHarness() { const state = { clearCalls: 0, @@ -204,7 +216,7 @@ describe("live test progress", () => { host, "e2e-openclaw-turn-latency", "openclaw", - "secret-api-key", + fakeInference(), undefined, progress, ); @@ -244,7 +256,7 @@ describe("live test progress", () => { host, "e2e-openclaw-turn-latency", "openclaw", - "secret-api-key", + fakeInference(), cleanupBeforeRetry, progress, ); @@ -298,7 +310,7 @@ describe("live test progress", () => { host, "e2e-openclaw-turn-latency", "openclaw", - "secret-api-key", + fakeInference(), cleanupBeforeRetry, progress, ), @@ -329,7 +341,7 @@ describe("live test progress", () => { onOutput: vi.fn(), }; - await cleanupTurnSandboxes(host, sandbox, progress); + await cleanupTurnSandboxes(host, sandbox, fakeInference(), progress); expect(command).toHaveBeenCalledTimes(2); expect(openshell).toHaveBeenCalledTimes(4);