From a171b745f71205e1e0b49b44e07e0adb28a77cd4 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 20 Aug 2026 14:01:01 +0000 Subject: [PATCH] feat(project): add online-eval to project add --- src/core/project/manager.tsx | 3 + src/handlers/project/add/index.ts | 2 + .../project/add/online-eval/index.test.ts | 271 ++++++++++++++++++ src/handlers/project/add/online-eval/index.ts | 93 ++++++ src/handlers/project/types.ts | 5 + 5 files changed, 374 insertions(+) create mode 100644 src/handlers/project/add/online-eval/index.test.ts create mode 100644 src/handlers/project/add/online-eval/index.ts diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 9ec02dd04..21eca81b4 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -161,6 +161,7 @@ export class FsProjectManager implements ProjectManager { ); } case "config-bundle": + case "online-eval": newResources.push(resourceConfig); break; @@ -289,5 +290,7 @@ function toProjectSpecKey(resourceType: ProjectResource) { return "runtimes"; case "config-bundle": return "configBundles"; + case "online-eval": + return "onlineEvalConfigs"; } } diff --git a/src/handlers/project/add/index.ts b/src/handlers/project/add/index.ts index 859bc4cd7..50a7e1a2f 100644 --- a/src/handlers/project/add/index.ts +++ b/src/handlers/project/add/index.ts @@ -2,6 +2,7 @@ import { withProject } from "../../../middleware/"; import { Router } from "../../../router"; import { createAddConfigBundleHandler } from "./config-bundle"; import { createAddHarnessHandler } from "./harness"; +import { createAddOnlineEvalHandler } from "./online-eval"; import type { AddProjectResourceConfig } from "./types"; export function createAddProjectResourceHandler(config: AddProjectResourceConfig): Router { @@ -9,5 +10,6 @@ export function createAddProjectResourceHandler(config: AddProjectResourceConfig projectAdd.use(withProject({ projectManager: config.projectManager, cwd: process.cwd() })); projectAdd.handler(createAddConfigBundleHandler(config)); projectAdd.handler(createAddHarnessHandler(config)); + projectAdd.handler(createAddOnlineEvalHandler(config)); return projectAdd; } diff --git a/src/handlers/project/add/online-eval/index.test.ts b/src/handlers/project/add/online-eval/index.test.ts new file mode 100644 index 000000000..c27453901 --- /dev/null +++ b/src/handlers/project/add/online-eval/index.test.ts @@ -0,0 +1,271 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../testing"; +import { DeserializationError, InputValidationError } from "../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-online-eval-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[], opts?: { core?: TestCoreClient }) { + const io = testIO(); + const core = opts?.core ?? new TestCoreClient(); + const root = createRootHandler(core, { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io, core }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +describe("project add online-eval", () => { + test.each<[string, string[], Record]>([ + [ + "minimal — agent source", + [ + "--name", + "x", + "--agent", + "hello_world", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "50", + ], + { agent: "hello_world", evaluators: ["Builtin.Correctness"], samplingRate: 50 }, + ], + [ + "custom log-group source", + [ + "--name", + "x", + "--log-group-name", + "/aws/foo", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "50", + ], + { logGroupNames: ["/aws/foo"], evaluators: ["Builtin.Correctness"], samplingRate: 50 }, + ], + [ + "agent source with endpoint", + [ + "--name", + "x", + "--agent", + "hello_world", + "--endpoint", + "PROD", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "10", + ], + { agent: "hello_world", endpoint: "PROD" }, + ], + [ + "service-name filter on a custom source", + [ + "--name", + "x", + "--log-group-name", + "/aws/foo", + "--service-name", + "svc", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "25", + ], + { logGroupNames: ["/aws/foo"], serviceNames: ["svc"] }, + ], + [ + "description", + [ + "--name", + "x", + "--log-group-name", + "/aws/foo", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "5", + "--description", + "monitor prod", + ], + { description: "monitor prod" }, + ], + [ + "enable-on-create false", + [ + "--name", + "x", + "--agent", + "hello_world", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "5", + "--enable-on-create", + "false", + ], + { enableOnCreate: false }, + ], + [ + "tags", + [ + "--name", + "x", + "--log-group-name", + "/aws/foo", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "5", + "--tags", + '{"team":"ml"}', + ], + { tags: { team: "ml" } }, + ], + ])("%s", async (_label, flags, expected) => { + const projectRoot = await inProject(); + await run(["add", "online-eval", ...flags]); + + const agentcoreJson = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const config = agentcoreJson.onlineEvalConfigs.find((c: { name: string }) => c.name === "x"); + expect(config).toMatchObject(expected); + }); + + test("rejects a duplicate online-eval name", async () => { + await inProject(); + const flags = [ + "--name", + "x", + "--log-group-name", + "/aws/foo", + "--evaluator", + "Builtin.Correctness", + "--sampling-rate", + "50", + ]; + await run(["add", "online-eval", ...flags]); + await expect(run(["add", "online-eval", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); + + test("rejects when the existing spec is invalid", async () => { + const projectRoot = await inProject(); + + const specPath = join(projectRoot, "agentcore", "agentcore.json"); + const spec = await Bun.file(specPath).json(); + spec.unknownField = "bad"; + await Bun.write(specPath, JSON.stringify(spec)); + + await expect( + run([ + "add", + "online-eval", + "--name", + "x", + "--agent", + "a", + "--evaluator", + "e", + "--sampling-rate", + "5", + ]), + ).rejects.toBeInstanceOf(DeserializationError); + }); + + test.each<[string, string[]]>([ + ["missing --name", ["--log-group-name", "/x", "--evaluator", "e", "--sampling-rate", "10"]], + ["missing --sampling-rate", ["--name", "x", "--log-group-name", "/x", "--evaluator", "e"]], + [ + "--agent and --log-group-name are mutually exclusive", + [ + "--name", + "x", + "--agent", + "a", + "--log-group-name", + "/x", + "--evaluator", + "e", + "--sampling-rate", + "10", + ], + ], + [ + "neither --agent nor --log-group-name", + ["--name", "x", "--evaluator", "e", "--sampling-rate", "10"], + ], + ["no evaluator", ["--name", "x", "--agent", "a", "--sampling-rate", "10"]], + [ + "--endpoint without --agent", + [ + "--name", + "x", + "--log-group-name", + "/x", + "--endpoint", + "PROD", + "--evaluator", + "e", + "--sampling-rate", + "10", + ], + ], + [ + "--service-name without --log-group-name", + [ + "--name", + "x", + "--agent", + "a", + "--service-name", + "svc", + "--evaluator", + "e", + "--sampling-rate", + "10", + ], + ], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "online-eval", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); +}); diff --git a/src/handlers/project/add/online-eval/index.ts b/src/handlers/project/add/online-eval/index.ts new file mode 100644 index 000000000..13b2ea3dc --- /dev/null +++ b/src/handlers/project/add/online-eval/index.ts @@ -0,0 +1,93 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { OnlineEvalConfigSchema } from "../../../../projectSchemas/online-eval-config"; +import { parseJsonFlag } from "../../../utils"; +import type { AddProjectResourceConfig } from "../types"; + +export const createAddOnlineEvalHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "online-eval", + description: "adds an online evaluation config to the current project", + flags: [ + flag("name", "the name of the online evaluation config", z.string().optional()), + flag( + "agent", + "harness/runtime name whose traffic to sample (mutually exclusive with --log-group-name)", + z.string().optional(), + ), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (requires --agent)", + z.string().optional(), + ), + flag( + "log-group-name", + "CloudWatch log group name(s) for custom data sources (1-5; mutually exclusive with --agent)", + z.array(z.string()).optional(), + ), + flag( + "service-name", + "service name(s) to filter traces for custom data sources (requires --log-group-name)", + z.array(z.string()).optional(), + ), + flag( + "evaluator", + "evaluator name(s), Builtin.* IDs, or ARNs to apply", + z.array(z.string()).optional(), + ), + flag( + "sampling-rate", + "percentage of sessions to sample (0.01-100)", + z.number().min(0.01).max(100).optional(), + ), + flag( + "description", + "a description of the config's monitoring purpose", + z.string().optional(), + ), + flag( + "enable-on-create", + "enable evaluation immediately after deploy (default true; pass false to add it paused)", + z.enum(["true", "false"]).optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (flags["sampling-rate"] === undefined) + throw new InputValidationError( + "required option '--sampling-rate ' not specified", + ); + + const candidate = { + name: flags["name"], + agent: flags["agent"], + endpoint: flags["endpoint"], + logGroupNames: flags["log-group-name"], + serviceNames: flags["service-name"], + evaluators: flags["evaluator"], + samplingRate: flags["sampling-rate"], + description: flags["description"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + tags: parseJsonFlag>("tags", flags["tags"]), + }; + + const parsed = OnlineEvalConfigSchema.safeParse(candidate); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + + const project = ctx.require(ProjectKey); + for await (const event of config.projectManager.addResource(project, { + resourceType: "online-eval", + resourceConfig: parsed.data, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added online-eval config '${flags["name"]}' to '${project.name}'\n`); + }, + }); diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index 24da19e94..560bf0080 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -3,6 +3,7 @@ import type { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import type { ProjectSpecSchema } from "../../projectSchemas/project"; import type z from "zod"; import type { ProjectRuntimeSchema } from "../../projectSchemas/runtime"; +import type { OnlineEvalConfigSchema } from "../../projectSchemas/online-eval-config"; /** Available project templates for scaffolding new AgentCore projects. */ export const PROJECT_TEMPLATES = { @@ -54,6 +55,10 @@ export type AddResourceInput = | { resourceType: "config-bundle"; resourceConfig: z.input; + } + | { + resourceType: "online-eval"; + resourceConfig: z.input; }; export type ProjectResource = AddResourceInput["resourceType"];