feat(project): add online-eval to project add

src/core/project/manager.tsx

@@ -161,6 +161,7 @@ export class FsProjectManager implements ProjectManager {
         );
       }
       case "config-bundle":
+      case "online-eval":
         newResources.push(resourceConfig);
         break;
 
@@ -289,5 +290,7 @@ function toProjectSpecKey(resourceType: ProjectResource) {
       return "runtimes";
     case "config-bundle":
       return "configBundles";
+    case "online-eval":
+      return "onlineEvalConfigs";
   }
 }

src/handlers/project/add/index.ts

@@ -2,12 +2,14 @@ import { withProject } from "../../../middleware/";
 import { Router } from "../../../router";
 import { createAddConfigBundleHandler } from "./config-bundle";
 import { createAddHarnessHandler } from "./harness";
+import { createAddOnlineEvalHandler } from "./online-eval";
 import type { AddProjectResourceConfig } from "./types";
 
 export function createAddProjectResourceHandler(config: AddProjectResourceConfig): Router {
   const projectAdd = new Router("add", "add project resources");
   projectAdd.use(withProject({ projectManager: config.projectManager, cwd: process.cwd() }));
   projectAdd.handler(createAddConfigBundleHandler(config));
   projectAdd.handler(createAddHarnessHandler(config));
+  projectAdd.handler(createAddOnlineEvalHandler(config));
   return projectAdd;
 }

src/handlers/project/add/online-eval/index.test.ts

@@ -0,0 +1,271 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { createRootHandler } from "../../../index";
+import {
+  createSilentLogger,
+  TestCoreClient,
+  TestGlobalConfigAccessor,
+  testIO,
+} from "../../../../testing";
+import { DeserializationError, InputValidationError } from "../../../../errors";
+
+const originalCwd = process.cwd();
+const tempDirectories: string[] = [];
+
+async function inTempDirectory(): Promise<string> {
+  const directory = await mkdtemp(join(tmpdir(), "agentcore-online-eval-"));
+  tempDirectories.push(directory);
+  process.chdir(directory);
+  return process.cwd();
+}
+
+afterEach(async () => {
+  process.chdir(originalCwd);
+  await Promise.all(
+    tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })),
+  );
+});
+
+async function run(args: string[], opts?: { core?: TestCoreClient }) {
+  const io = testIO();
+  const core = opts?.core ?? new TestCoreClient();
+  const root = createRootHandler(core, {
+    io: io.io,
+    globalConfigAccessor: new TestGlobalConfigAccessor(),
+    logger: createSilentLogger(),
+  });
+  await root.route(["node", "agentcore", "project", ...args]);
+  return { io, core };
+}
+
+async function inProject(name = "TestProject"): Promise<string> {
+  const directory = await inTempDirectory();
+  await run(["create", "--name", name, "--skip-install", "--skip-git"]);
+  const projectRoot = join(directory, name);
+  process.chdir(projectRoot);
+  return projectRoot;
+}
+
+describe("project add online-eval", () => {
+  test.each<[string, string[], Record<string, unknown>]>([
+    [
+      "minimal — agent source",
+      [
+        "--name",
+        "x",
+        "--agent",
+        "hello_world",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "50",
+      ],
+      { agent: "hello_world", evaluators: ["Builtin.Correctness"], samplingRate: 50 },
+    ],
+    [
+      "custom log-group source",
+      [
+        "--name",
+        "x",
+        "--log-group-name",
+        "/aws/foo",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "50",
+      ],
+      { logGroupNames: ["/aws/foo"], evaluators: ["Builtin.Correctness"], samplingRate: 50 },
+    ],
+    [
+      "agent source with endpoint",
+      [
+        "--name",
+        "x",
+        "--agent",
+        "hello_world",
+        "--endpoint",
+        "PROD",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "10",
+      ],
+      { agent: "hello_world", endpoint: "PROD" },
+    ],
+    [
+      "service-name filter on a custom source",
+      [
+        "--name",
+        "x",
+        "--log-group-name",
+        "/aws/foo",
+        "--service-name",
+        "svc",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "25",
+      ],
+      { logGroupNames: ["/aws/foo"], serviceNames: ["svc"] },
+    ],
+    [
+      "description",
+      [
+        "--name",
+        "x",
+        "--log-group-name",
+        "/aws/foo",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "5",
+        "--description",
+        "monitor prod",
+      ],
+      { description: "monitor prod" },
+    ],
+    [
+      "enable-on-create false",
+      [
+        "--name",
+        "x",
+        "--agent",
+        "hello_world",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "5",
+        "--enable-on-create",
+        "false",
+      ],
+      { enableOnCreate: false },
+    ],
+    [
+      "tags",
+      [
+        "--name",
+        "x",
+        "--log-group-name",
+        "/aws/foo",
+        "--evaluator",
+        "Builtin.Correctness",
+        "--sampling-rate",
+        "5",
+        "--tags",
+        '{"team":"ml"}',
+      ],
+      { tags: { team: "ml" } },
+    ],
+  ])("%s", async (_label, flags, expected) => {
+    const projectRoot = await inProject();
+    await run(["add", "online-eval", ...flags]);
+
+    const agentcoreJson = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json();
+    const config = agentcoreJson.onlineEvalConfigs.find((c: { name: string }) => c.name === "x");
+    expect(config).toMatchObject(expected);
+  });
+
+  test("rejects a duplicate online-eval name", async () => {
+    await inProject();
+    const flags = [
+      "--name",
+      "x",
+      "--log-group-name",
+      "/aws/foo",
+      "--evaluator",
+      "Builtin.Correctness",
+      "--sampling-rate",
+      "50",
+    ];
+    await run(["add", "online-eval", ...flags]);
+    await expect(run(["add", "online-eval", ...flags])).rejects.toBeInstanceOf(
+      InputValidationError,
+    );
+  });
+
+  test("rejects when the existing spec is invalid", async () => {
+    const projectRoot = await inProject();
+
+    const specPath = join(projectRoot, "agentcore", "agentcore.json");
+    const spec = await Bun.file(specPath).json();
+    spec.unknownField = "bad";
+    await Bun.write(specPath, JSON.stringify(spec));
+
+    await expect(
+      run([
+        "add",
+        "online-eval",
+        "--name",
+        "x",
+        "--agent",
+        "a",
+        "--evaluator",
+        "e",
+        "--sampling-rate",
+        "5",
+      ]),
+    ).rejects.toBeInstanceOf(DeserializationError);
+  });
+
+  test.each<[string, string[]]>([
+    ["missing --name", ["--log-group-name", "/x", "--evaluator", "e", "--sampling-rate", "10"]],
+    ["missing --sampling-rate", ["--name", "x", "--log-group-name", "/x", "--evaluator", "e"]],
+    [
+      "--agent and --log-group-name are mutually exclusive",
+      [
+        "--name",
+        "x",
+        "--agent",
+        "a",
+        "--log-group-name",
+        "/x",
+        "--evaluator",
+        "e",
+        "--sampling-rate",
+        "10",
+      ],
+    ],
+    [
+      "neither --agent nor --log-group-name",
+      ["--name", "x", "--evaluator", "e", "--sampling-rate", "10"],
+    ],
+    ["no evaluator", ["--name", "x", "--agent", "a", "--sampling-rate", "10"]],
+    [
+      "--endpoint without --agent",
+      [
+        "--name",
+        "x",
+        "--log-group-name",
+        "/x",
+        "--endpoint",
+        "PROD",
+        "--evaluator",
+        "e",
+        "--sampling-rate",
+        "10",
+      ],
+    ],
+    [
+      "--service-name without --log-group-name",
+      [
+        "--name",
+        "x",
+        "--agent",
+        "a",
+        "--service-name",
+        "svc",
+        "--evaluator",
+        "e",
+        "--sampling-rate",
+        "10",
+      ],
+    ],
+  ])("%s", async (_label, flags) => {
+    await inProject();
+    await expect(run(["add", "online-eval", ...flags])).rejects.toBeInstanceOf(
+      InputValidationError,
+    );
+  });
+});

src/handlers/project/add/online-eval/index.ts

@@ -0,0 +1,93 @@
+import z from "zod";
+import { createHandler, flag, ProjectKey } from "../../../../router";
+import { InputValidationError } from "../../../../errors";
+import { OnlineEvalConfigSchema } from "../../../../projectSchemas/online-eval-config";
+import { parseJsonFlag } from "../../../utils";
+import type { AddProjectResourceConfig } from "../types";
+
+export const createAddOnlineEvalHandler = (config: AddProjectResourceConfig) =>
+  createHandler({
+    name: "online-eval",
+    description: "adds an online evaluation config to the current project",
+    flags: [
+      flag("name", "the name of the online evaluation config", z.string().optional()),
+      flag(
+        "agent",
+        "harness/runtime name whose traffic to sample (mutually exclusive with --log-group-name)",
+        z.string().optional(),
+      ),
+      flag(
+        "endpoint",
+        "the agent endpoint qualifier to scope monitoring to (requires --agent)",
+        z.string().optional(),
+      ),
+      flag(
+        "log-group-name",
+        "CloudWatch log group name(s) for custom data sources (1-5; mutually exclusive with --agent)",
+        z.array(z.string()).optional(),
+      ),
+      flag(
+        "service-name",
+        "service name(s) to filter traces for custom data sources (requires --log-group-name)",
+        z.array(z.string()).optional(),
+      ),
+      flag(
+        "evaluator",
+        "evaluator name(s), Builtin.* IDs, or ARNs to apply",
+        z.array(z.string()).optional(),
+      ),
+      flag(
+        "sampling-rate",
+        "percentage of sessions to sample (0.01-100)",
+        z.number().min(0.01).max(100).optional(),
+      ),
+      flag(
+        "description",
+        "a description of the config's monitoring purpose",
+        z.string().optional(),
+      ),
+      flag(
+        "enable-on-create",
+        "enable evaluation immediately after deploy (default true; pass false to add it paused)",
+        z.enum(["true", "false"]).optional(),
+      ),
+      flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()),
+    ],
+    handle: async (ctx, flags) => {
+      if (!flags["name"])
+        throw new InputValidationError("required option '--name <name>' not specified");
+      if (flags["sampling-rate"] === undefined)
+        throw new InputValidationError(
+          "required option '--sampling-rate <sampling-rate>' not specified",
+        );
+
+      const candidate = {
+        name: flags["name"],
+        agent: flags["agent"],
+        endpoint: flags["endpoint"],
+        logGroupNames: flags["log-group-name"],
+        serviceNames: flags["service-name"],
+        evaluators: flags["evaluator"],
+        samplingRate: flags["sampling-rate"],
+        description: flags["description"],
+        enableOnCreate:
+          flags["enable-on-create"] === undefined
+            ? undefined
+            : flags["enable-on-create"] === "true",
+        tags: parseJsonFlag<Record<string, string>>("tags", flags["tags"]),
+      };
+
+      const parsed = OnlineEvalConfigSchema.safeParse(candidate);
+      if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error));
+
+      const project = ctx.require(ProjectKey);
+      for await (const event of config.projectManager.addResource(project, {
+        resourceType: "online-eval",
+        resourceConfig: parsed.data,
+      })) {
+        config.io.stderr.write(`${event.message}\n`);
+      }
+
+      config.io.stderr.write(`added online-eval config '${flags["name"]}' to '${project.name}'\n`);
+    },
+  });