diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 037ec7567f4..92337ef5c45 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2732,6 +2732,8 @@ OpenClaw caches skill content per session, so the command also refreshes the Ope Hermes plugins are different from NemoClaw skills. `skill install` uploads agent skills, while Hermes plugin configuration is managed by the Hermes runtime and the NemoClaw Hermes plugin baked into the sandbox image. +The NemoClaw Hermes plugin reloads installed skills when a new chat session starts. +Start a new Hermes chat session after an install or update; a gateway restart is not required. @@ -2778,7 +2780,8 @@ For OpenClaw, the command also removes the OpenClaw home-directory mirror when p -Run `$$nemoclaw gateway restart` if prompted so the removal takes effect. +Start a new Hermes chat session for the removal to take effect. +A gateway restart is not required. diff --git a/src/lib/actions/sandbox/skill-install.test.ts b/src/lib/actions/sandbox/skill-install.test.ts index 5e9a2fc1359..a5d84afd8be 100644 --- a/src/lib/actions/sandbox/skill-install.test.ts +++ b/src/lib/actions/sandbox/skill-install.test.ts @@ -45,6 +45,7 @@ const paths = { mirrorDir: "$HOME/.openclaw/skills/demo-skill", uploadDirSharedWithAgent: false, sessionFile: "/sandbox/.openclaw/agents/main/sessions/sessions.json", + reloadsSkillsOnSessionStart: false, isOpenClaw: true, }; @@ -59,6 +60,7 @@ const sharedPaths = { mirrorDir: null, uploadDirSharedWithAgent: true, sessionFile: null, + reloadsSkillsOnSessionStart: false, isOpenClaw: false, }; diff --git a/src/lib/actions/sandbox/skill-install.ts b/src/lib/actions/sandbox/skill-install.ts index 692fe449c79..f2196993158 100644 --- a/src/lib/actions/sandbox/skill-install.ts +++ b/src/lib/actions/sandbox/skill-install.ts @@ -423,7 +423,7 @@ export async function installSandboxSkill( } console.log(` ${G}✓${R} Uploaded ${uploaded} file(s) to sandbox`); - // 7. Post-install (OpenClaw mirror + refresh, or restart hint). + // 7. Post-install (OpenClaw mirror + refresh, or agent-specific activation guidance). // OpenClaw caches skill content per session, so always refresh the // session index after an install/update to avoid stale SKILL.md data. const post = skillInstall.postInstall(ctx, paths, skillDir); diff --git a/src/lib/skill-install.test.ts b/src/lib/skill-install.test.ts index 3a6a1585359..89f3f2f3c0e 100644 --- a/src/lib/skill-install.test.ts +++ b/src/lib/skill-install.test.ts @@ -218,6 +218,7 @@ describe("resolveSkillPaths", () => { expect(paths.mirrorDir).toBe("$HOME/.openclaw/skills/weather"); expect(paths.uploadDirSharedWithAgent).toBe(false); expect(paths.sessionFile).toBe("/sandbox/.openclaw/agents/main/sessions/sessions.json"); + expect(paths.reloadsSkillsOnSessionStart).toBe(false); expect(paths.isOpenClaw).toBe(true); }); @@ -234,6 +235,7 @@ describe("resolveSkillPaths", () => { expect(paths.mirrorDir).toBe("$HOME/.openclaw/skills/my-skill"); expect(paths.uploadDirSharedWithAgent).toBe(false); expect(paths.sessionFile).toBe("/sandbox/.openclaw/agents/main/sessions/sessions.json"); + expect(paths.reloadsSkillsOnSessionStart).toBe(false); expect(paths.isOpenClaw).toBe(true); }); @@ -250,6 +252,7 @@ describe("resolveSkillPaths", () => { expect(paths.mirrorDir).toBeNull(); expect(paths.uploadDirSharedWithAgent).toBe(false); expect(paths.sessionFile).toBeNull(); + expect(paths.reloadsSkillsOnSessionStart).toBe(true); expect(paths.isOpenClaw).toBe(false); }); @@ -269,6 +272,7 @@ describe("resolveSkillPaths", () => { expect(paths.mirrorDir).toBeNull(); expect(paths.uploadDirSharedWithAgent).toBe(true); expect(paths.sessionFile).toBeNull(); + expect(paths.reloadsSkillsOnSessionStart).toBe(false); expect(paths.isOpenClaw).toBe(false); }); @@ -285,11 +289,34 @@ describe("resolveSkillPaths", () => { expect(paths.mirrorDir).toBeNull(); expect(paths.uploadDirSharedWithAgent).toBe(false); expect(paths.sessionFile).toBeNull(); + expect(paths.reloadsSkillsOnSessionStart).toBe(false); expect(paths.isOpenClaw).toBe(false); }); }); describe("postInstall", () => { + it("tells Hermes users to start a fresh session without restarting the gateway", () => { + const paths = resolveSkillPaths( + { name: "hermes", configPaths: { dir: "/sandbox/.hermes" } }, + "weather", + ); + const result = postInstall( + { configFile: "/tmp/ssh-config", sandboxName: "alpha" }, + paths, + "/unused", + { + sshExecImpl: () => { + throw new Error("Hermes activation must not require an SSH mutation"); + }, + }, + ); + + expect(result).toEqual({ + success: true, + messages: ["Start a new chat session to load the skill; a gateway restart is not required."], + }); + }); + it("refreshes OpenClaw sessions after installing an updated skill", () => { const skillDir = mkdtempSync(join(tmpdir(), "skill-postinstall-")); const commands: string[] = []; diff --git a/src/lib/skill-install.ts b/src/lib/skill-install.ts index a8da5d48c6f..ae820cdad4e 100644 --- a/src/lib/skill-install.ts +++ b/src/lib/skill-install.ts @@ -111,6 +111,8 @@ export interface SkillPaths { uploadDirSharedWithAgent: boolean; /** OpenClaw-only: session index to clear, or null */ sessionFile: string | null; + /** Whether a fresh agent session reloads skills without a gateway restart */ + reloadsSkillsOnSessionStart: boolean; /** Whether the agent is OpenClaw (drives refresh behavior) */ isOpenClaw: boolean; } @@ -164,6 +166,7 @@ export function resolveSkillPaths( mirrorDir: mirror ? mirror(dir, skillName) : null, uploadDirSharedWithAgent: Boolean(sharedDir), sessionFile: isOpenClaw ? `${dir}/agents/main/sessions/sessions.json` : null, + reloadsSkillsOnSessionStart: agentName === "hermes", isOpenClaw, }; } @@ -597,7 +600,7 @@ export function installFreshSharedSkill( /** * Run post-install steps: skill-load mirror for every agent that needs one, - * session refresh for OpenClaw, and a restart hint when neither applies. + * session refresh for OpenClaw, and agent-specific activation guidance. */ export function postInstall( ctx: SshContext, @@ -645,7 +648,11 @@ export function postInstall( } if (!paths.mirrorDir && !paths.sessionFile) { - messages.push("Restart the agent gateway to pick up the new skill."); + messages.push( + paths.reloadsSkillsOnSessionStart + ? "Start a new chat session to load the skill; a gateway restart is not required." + : "Restart the agent gateway to pick up the new skill.", + ); } return { success: true, messages }; diff --git a/src/lib/skill-remote.test.ts b/src/lib/skill-remote.test.ts index fd6d12b6ec3..759a9b43ff6 100644 --- a/src/lib/skill-remote.test.ts +++ b/src/lib/skill-remote.test.ts @@ -85,6 +85,27 @@ describe("removeSkill (unit — no SSH)", () => { ]); }); + it("tells Hermes users to start a fresh session after removal", () => { + const ctx = { configFile: "/tmp/ssh.conf", sandboxName: "test-sandbox" }; + const paths = resolveSkillPaths( + { name: "hermes", configPaths: { dir: "/sandbox/.hermes" } }, + "test-skill", + ); + const commands: string[] = []; + const result = removeSkill(ctx, paths, { + sshExecImpl: (_ctx, command) => { + commands.push(command); + return { status: 0, stdout: "", stderr: "" }; + }, + }); + + expect(result.success).toBe(true); + expect(result.messages).toEqual([ + "Start a new chat session for the removal to take effect; a gateway restart is not required.", + ]); + expect(commands).toEqual(["rm -rf '/sandbox/.hermes/skills/test-skill'"]); + }); + it("probes the canonical Deep Agents directory for diagnostics (#7634)", () => { const ctx = { configFile: "/tmp/ssh.conf", sandboxName: "test-sandbox" }; const paths = resolveSkillPaths( diff --git a/src/lib/skill-remote.ts b/src/lib/skill-remote.ts index 62369417d4c..c02340924f6 100644 --- a/src/lib/skill-remote.ts +++ b/src/lib/skill-remote.ts @@ -166,7 +166,11 @@ export function removeSkill( } if (!paths.mirrorDir && !paths.sessionFile) { - messages.push("Restart the agent gateway for the removal to take effect."); + messages.push( + paths.reloadsSkillsOnSessionStart + ? "Start a new chat session for the removal to take effect; a gateway restart is not required." + : "Restart the agent gateway for the removal to take effect.", + ); } return { diff --git a/test/e2e/fixtures/fake-openai-compatible.ts b/test/e2e/fixtures/fake-openai-compatible.ts index 5e0ec5b5e5d..6e4c8d85cda 100644 --- a/test/e2e/fixtures/fake-openai-compatible.ts +++ b/test/e2e/fixtures/fake-openai-compatible.ts @@ -23,6 +23,8 @@ export interface FakeOpenAiCompatibleRequest { readonly model?: string; readonly stream?: boolean; readonly forbiddenMarkerMatches?: number; + /** Presence only; the configured non-secret canary is never persisted. */ + readonly requestCanaryPresent?: boolean; } export interface FakeOpenAiCompatibleServer { @@ -38,6 +40,8 @@ export interface FakeOpenAiCompatibleServerOptions { readonly apiKey?: string; readonly chatContent?: string; readonly forbiddenMarkers?: readonly string[]; + /** Non-secret marker expected in a request under test. */ + readonly requestCanaryMarker?: string; readonly host?: string; readonly maxModelLen?: number; readonly model?: string; @@ -185,6 +189,7 @@ export async function startFakeOpenAiCompatibleServer( NEMOCLAW_FAKE_OPENAI_MODEL: options.model ?? "test-model", NEMOCLAW_FAKE_OPENAI_PORT: String(options.port ?? 0), NEMOCLAW_FAKE_OPENAI_PORT_FILE: portFile, + NEMOCLAW_FAKE_OPENAI_REQUEST_CANARY_MARKER: options.requestCanaryMarker ?? "", NEMOCLAW_FAKE_OPENAI_REQUESTS_FILE: requestsFile, NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH: options.requireAuth ? "1" : "0", NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH_MODELS: options.requireAuthModels ? "1" : "0", diff --git a/test/e2e/fixtures/hermes-skill-runtime/SKILL.md b/test/e2e/fixtures/hermes-skill-runtime/SKILL.md new file mode 100644 index 00000000000..5a5f91fdb0c --- /dev/null +++ b/test/e2e/fixtures/hermes-skill-runtime/SKILL.md @@ -0,0 +1,16 @@ +--- +name: nemoclaw-hermes-skill-e2e +description: Verifies Hermes skill discovery and fresh-session execution +--- + + + + +# Hermes skill runtime verification + +When this skill is selected, do not use tools. Reply with exactly `PONG` and nothing else. + +The following non-secret test canary must remain in the model-visible skill content: +`NEMOCLAW_E2E_REQUEST_CANARY_K9X2` + +Do not include the canary in your response. diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index 8410da56b25..918e6b8460f 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -7,6 +7,7 @@ import type { ArtifactSink } from "./artifacts.ts"; import { buildAvailabilityProbeEnv } from "./availability-env.ts"; import { type ProviderClient, trustedProviderEndpoint } from "./clients/provider.ts"; import { + type FakeOpenAiCompatibleRequest, type FakeOpenAiCompatibleServer, startFakeOpenAiCompatibleServer, } from "./fake-openai-compatible.ts"; @@ -42,6 +43,9 @@ import type { TestProgress, TestProgressCapability } from "./progress.ts"; export const E2E_INFERENCE_MODE_VALUES = ["mock", "internal-nvidia", "public-nvidia"] as const; export type E2EInferenceMode = (typeof E2E_INFERENCE_MODE_VALUES)[number]; +/** Non-secret marker used to prove fixture content reached the local mock inference boundary. */ +export const E2E_MOCK_REQUEST_CANARY = "NEMOCLAW_E2E_REQUEST_CANARY_K9X2"; + export interface E2EInferenceAdapter { readonly mode: E2EInferenceMode; readonly model: string; @@ -52,6 +56,8 @@ export interface E2EInferenceAdapter { readonly contractLabel: string; env(extra?: NodeJS.ProcessEnv): NodeJS.ProcessEnv; redactionValues(): string[]; + /** Privacy-safe mock request metadata; unavailable when inference is hosted. */ + requestSummaries(): readonly FakeOpenAiCompatibleRequest[] | undefined; probeModels(artifactName: string): Promise; directChat( prompt: string, @@ -219,6 +225,10 @@ class OpenAiCompatibleInferenceAdapter implements E2EInferenceAdapter { return [this.apiKey]; } + requestSummaries(): readonly FakeOpenAiCompatibleRequest[] | undefined { + return this.fake?.requests(); + } + async probeModels(artifactName: string): Promise { if (this.providerClient) { return requestViaProvider(this.providerClient, { @@ -326,6 +336,10 @@ class PublicNvidiaInferenceAdapter implements E2EInferenceAdapter { return [this.apiKey]; } + requestSummaries(): undefined { + return undefined; + } + async probeModels(artifactName: string): Promise { return requestViaProvider(this.providerClient, { allowedHosts: PUBLIC_NVIDIA_ALLOWED_HOSTS, @@ -370,6 +384,8 @@ export async function createE2EInferenceAdapter( const fake = await startFakeOpenAiCompatibleServer({ apiKey, chatContent: "PONG", + // The fake stores only presence metadata, never request bodies or the canary value. + requestCanaryMarker: E2E_MOCK_REQUEST_CANARY, // A Docker network namespace cannot reach host loopback through the host // alias, so listen on the bridge-facing interfaces. The workflow uses an // ephemeral ubuntu-latest VM, an OS-assigned port, and a per-run credential. diff --git a/test/e2e/lib/fake-openai-compatible-api.mts b/test/e2e/lib/fake-openai-compatible-api.mts index 24567e157b5..4af3b7378d6 100755 --- a/test/e2e/lib/fake-openai-compatible-api.mts +++ b/test/e2e/lib/fake-openai-compatible-api.mts @@ -28,6 +28,7 @@ const requireAuth = process.env.NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH === "1"; const requireAuthModels = process.env.NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH_MODELS === "1"; const chatContent = process.env.NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT || "ok"; const responseText = process.env.NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT || chatContent; +const requestCanaryMarker = process.env.NEMOCLAW_FAKE_OPENAI_REQUEST_CANARY_MARKER || ""; const forbiddenMarkers = (() => { try { const parsed = JSON.parse(process.env.NEMOCLAW_FAKE_OPENAI_FORBIDDEN_MARKERS || "[]"); @@ -137,6 +138,13 @@ function forbiddenMarkerMatches(req: IncomingMessage, raw: Buffer): number { return forbiddenMarkers.filter((marker) => requestMaterial.includes(marker)).length; } +function requestCanaryPresent(req: IncomingMessage, raw: Buffer): boolean | undefined { + if (!requestCanaryMarker) return undefined; + const headerValues = Object.values(req.headers).flatMap((value) => value ?? []); + const requestMaterial = [req.url ?? "", ...headerValues, raw.toString("utf8")].join("\n"); + return requestMaterial.includes(requestCanaryMarker); +} + const server = createServer(async (req, res) => { const path = requestPath(req); @@ -153,6 +161,7 @@ const server = createServer(async (req, res) => { // credential without leaking it into the requests log (#6177). authorizationSent: Boolean(req.headers.authorization), forbiddenMarkerMatches: forbiddenMarkerMatches(req, Buffer.alloc(0)), + requestCanaryPresent: requestCanaryPresent(req, Buffer.alloc(0)), }); if (!modelsAuthOk) { sendJson(res, 401, { error: { message: "missing bearer credential" } }); @@ -178,6 +187,7 @@ const server = createServer(async (req, res) => { model: payload.model, stream: Boolean(payload.stream), forbiddenMarkerMatches: forbiddenMarkerMatches(req, raw), + requestCanaryPresent: requestCanaryPresent(req, raw), }); if (req.method === "POST" && ["/v1/chat/completions", "/chat/completions"].includes(path)) { diff --git a/test/e2e/live/hermes-e2e-phases.ts b/test/e2e/live/hermes-e2e-phases.ts index e3cc35edf48..640bc94a19e 100644 --- a/test/e2e/live/hermes-e2e-phases.ts +++ b/test/e2e/live/hermes-e2e-phases.ts @@ -4,7 +4,7 @@ export const HERMES_E2E_PHASES = [ "prepare clean Hermes runner", "install and onboard Hermes sandbox", - "validate sandbox layout and health", + "validate sandbox layout, health, and skill activation", "restart Hermes gateway, validate supervision, and launch a turn", "exercise hosted and inference.local routes", "validate CLI manifest and locked-config behavior", diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 980502e82af..63b2f287ca0 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -19,6 +19,7 @@ import { import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { assertHermesCliAdapterLiveContract, stripAnsi } from "./hermes-cli-adapter-live.ts"; import { HERMES_E2E_PHASES } from "./hermes-e2e-phases.ts"; +import { assertHermesSkillLifecycle } from "./hermes-skill-lifecycle.ts"; import { runLaunchAgentTurn } from "./launch-agent-turn.ts"; import { expectPackageDatabaseReadOnly } from "./package-database-read-only.ts"; @@ -376,7 +377,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(resultText(install)).toContain(`http://127.0.0.1:${HERMES_DASHBOARD_PORT}/`); } - progress.phase("validate sandbox layout and health"); + progress.phase("validate sandbox layout, health, and skill activation"); // Phase 3: sandbox verification. const list = await host.command("nemoclaw", ["list"], { artifactName: "phase-3-nemoclaw-list", @@ -500,6 +501,14 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(configProbe.exitCode, resultText(configProbe)).toBe(0); expect(configProbe.stdout).toContain("OK"); + await assertHermesSkillLifecycle({ + env: commandEnv(), + host, + inference, + redactionValues, + sandboxName: SANDBOX_NAME, + }); + await assertHermesCliAdapterLiveContract({ env: commandEnv(), host, @@ -1444,6 +1453,9 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen sandboxListedAndHealthy: true, directProviderInferencePong: true, sandboxInferenceLocalPong: true, + hermesSkillInstalled: true, + hermesSkillDiscovered: true, + hermesSkillUsedInFreshSession: true, dashboardChecked: hermesDashboardE2eEnabled(), securityPostureChecked: securityPosture !== null, }, diff --git a/test/e2e/live/hermes-skill-lifecycle.ts b/test/e2e/live/hermes-skill-lifecycle.ts new file mode 100644 index 00000000000..2ae614bba2a --- /dev/null +++ b/test/e2e/live/hermes-skill-lifecycle.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { + E2E_MOCK_REQUEST_CANARY, + type E2EInferenceAdapter, +} from "../fixtures/inference-adapter.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import { hermesSessionIds, onlyNewHermesSessionId, stripAnsi } from "./hermes-cli-adapter-live.ts"; + +const HERMES_SKILL_ID = "nemoclaw-hermes-skill-e2e"; +const HERMES_SKILL_FIXTURE = path.join( + REPO_ROOT, + "test", + "e2e", + "fixtures", + "hermes-skill-runtime", +); +const HERMES_SKILL_PROMPT = + "Follow the selected verification skill and return only its verification value."; +const INFERENCE_REQUEST_PATHS = new Set([ + "/v1/chat/completions", + "/chat/completions", + "/v1/responses", + "/responses", +]); + +interface HermesSkillLifecycleOptions { + env: NodeJS.ProcessEnv; + host: HostCliClient; + inference: Pick; + redactionValues: string[]; + sandboxName: string; +} + +/** + * Prove the public Hermes skill lifecycle through NemoClaw without persisting + * inference request bodies. The local mock records only whether the skill's + * non-secret canary crossed the inference boundary. + */ +export async function assertHermesSkillLifecycle({ + env, + host, + inference, + redactionValues, + sandboxName, +}: HermesSkillLifecycleOptions): Promise { + const exec = async ( + args: string[], + artifactName: string, + remoteTimeoutSeconds = 60, + hostTimeoutMs = 90_000, + ) => { + const result = await host.command( + "nemohermes", + [sandboxName, "exec", "--no-stdin", "--timeout", String(remoteTimeoutSeconds), "--", ...args], + { artifactName, env, redactionValues, timeoutMs: hostTimeoutMs }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + return result; + }; + + const skillFixtureText = fs.readFileSync(path.join(HERMES_SKILL_FIXTURE, "SKILL.md"), "utf8"); + expect(skillFixtureText).toContain(E2E_MOCK_REQUEST_CANARY); + expect(HERMES_SKILL_PROMPT).not.toContain(E2E_MOCK_REQUEST_CANARY); + expect(HERMES_SKILL_PROMPT).not.toMatch(/PONG/i); + + const skillInstall = await host.command( + "nemohermes", + [sandboxName, "skill", "install", HERMES_SKILL_FIXTURE], + { + artifactName: "phase-4-hermes-skill-install", + cwd: REPO_ROOT, + env, + redactionValues, + timeoutMs: 120_000, + }, + ); + expect(skillInstall.exitCode, resultText(skillInstall)).toBe(0); + expect(stripAnsi(resultText(skillInstall))).toContain(`Skill '${HERMES_SKILL_ID}' installed`); + expect(stripAnsi(resultText(skillInstall))).toContain( + "Start a new chat session to load the skill; a gateway restart is not required.", + ); + + await exec( + ["test", "-f", `/sandbox/.hermes/skills/${HERMES_SKILL_ID}/SKILL.md`], + "phase-4-hermes-skill-disk-check", + ); + const skillList = await exec(["hermes", "skills", "list"], "phase-4-hermes-skills-list"); + expect(stripAnsi(resultText(skillList))).toContain(HERMES_SKILL_ID); + + const sessionsBeforeSkill = await exec( + ["hermes", "sessions", "list"], + "phase-4-hermes-skill-sessions-before", + ); + const requestOffset = inference.requestSummaries()?.length; + const skillChat = await exec( + ["hermes", "chat", "--skills", HERMES_SKILL_ID, "--query", HERMES_SKILL_PROMPT, "--quiet"], + "phase-4-hermes-skill-chat", + 360, + 420_000, + ); + expect(stripAnsi(resultText(skillChat))).toMatch(/\bPONG\b/i); + + const sessionsAfterSkill = await exec( + ["hermes", "sessions", "list"], + "phase-4-hermes-skill-sessions-after", + ); + expect( + onlyNewHermesSessionId( + hermesSessionIds(resultText(sessionsBeforeSkill)), + hermesSessionIds(resultText(sessionsAfterSkill)), + ), + ).toMatch(/^\d{8}_\d{6}_[a-zA-Z0-9]+$/); + + if (requestOffset === undefined) return; + const skillRequests = (inference.requestSummaries() ?? []) + .slice(requestOffset) + .filter((request) => request.method === "POST" && INFERENCE_REQUEST_PATHS.has(request.path)); + expect(skillRequests.length).toBeGreaterThan(0); + expect( + skillRequests.some((request) => request.auth === "ok" && request.requestCanaryPresent === true), + "installed Hermes skill canary did not reach an authenticated mock inference request", + ).toBe(true); +} diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index c98a88db437..0d6eba78a45 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -385,6 +385,7 @@ describe("hosted inference E2E config", () => { forbiddenMarkers: ["FORBIDDEN_REQUEST_MARKER"], model: "nvidia/nvidia/fake-model", progress, + requestCanaryMarker: "EXPECTED_REQUEST_CANARY", requireAuth: true, responseText: "RESP_OK", }); @@ -408,7 +409,7 @@ describe("hosted inference E2E config", () => { const chat = await fetch(`${fake.baseUrl}/chat/completions`, { body: JSON.stringify({ - messages: [{ content: "ping", role: "user" }], + messages: [{ content: "ping EXPECTED_REQUEST_CANARY", role: "user" }], model: "nvidia/nvidia/fake-model", }), headers: { @@ -443,6 +444,7 @@ describe("hosted inference E2E config", () => { auth: "missing", forbiddenMarkerMatches: 1, path: "/v1/chat/completions", + requestCanaryPresent: false, }), expect.objectContaining({ auth: "ok", @@ -450,6 +452,7 @@ describe("hosted inference E2E config", () => { hostHeader: new URL(fake.baseUrl).host, model: "nvidia/nvidia/fake-model", path: "/v1/chat/completions", + requestCanaryPresent: true, stream: false, }), expect.objectContaining({ auth: "ok", path: "/v1/responses", stream: true }), @@ -459,6 +462,7 @@ describe("hosted inference E2E config", () => { requests.reduce((total, request) => total + (request.forbiddenMarkerMatches ?? 0), 0), ).toBe(1); expect(JSON.stringify(requests)).not.toContain("FORBIDDEN_REQUEST_MARKER"); + expect(JSON.stringify(requests)).not.toContain("EXPECTED_REQUEST_CANARY"); } finally { await fake.close(); } diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index 6467943e793..02d2ee276ca 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -15,6 +15,7 @@ import type { import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { createE2EInferenceAdapter, + E2E_MOCK_REQUEST_CANARY, type E2EInferenceAdapter, requirePublicNvidiaInferenceKey, } from "../fixtures/inference-adapter.ts"; @@ -164,9 +165,19 @@ describe("E2E inference adapter", () => { expect(await adapter.probeModels("mock-models")).toMatchObject({ data: [{ id: "nvidia/nvidia/nemotron-3-ultra" }], }); - expect(await adapter.directChat("Reply PONG")).toMatchObject({ + const requestOffset = adapter.requestSummaries()?.length ?? 0; + expect(await adapter.directChat(`Reply PONG ${E2E_MOCK_REQUEST_CANARY}`)).toMatchObject({ choices: [{ message: { content: "PONG" } }], }); + expect(adapter.requestSummaries()?.slice(requestOffset)).toContainEqual( + expect.objectContaining({ + auth: "ok", + method: "POST", + path: "/v1/chat/completions", + forbiddenMarkerMatches: 0, + requestCanaryPresent: true, + }), + ); }); it("keeps unrelated ambient secrets out of adapter and fake-server child environments", async () => { @@ -207,6 +218,7 @@ describe("E2E inference adapter", () => { const env = adapter.env({ NVIDIA_INFERENCE_API_KEY: "ambient-source-key" }); expect(adapter.mode).toBe("internal-nvidia"); + expect(adapter.requestSummaries()).toBeUndefined(); expect(adapter.expectedRouteProvider).toBe("compatible-endpoint"); expect(env).toMatchObject({ NEMOCLAW_E2E_INFERENCE_MODE: "internal-nvidia", @@ -304,6 +316,7 @@ describe("E2E inference adapter", () => { }); expect(adapter.mode).toBe("public-nvidia"); + expect(adapter.requestSummaries()).toBeUndefined(); expect(adapter.expectedRouteProvider).toBe("nvidia-prod"); expect(adapter.endpointUrl).toBe("https://integrate.api.nvidia.com/v1"); expect(env).toMatchObject({