From ea55f45823fb78f53db3f4e61a4ab4d46294a793 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Thu, 24 Sep 2026 18:17:27 +0000 Subject: [PATCH 1/3] codeybox: Add unreal-agent (Unreal Labs) as a supported agent runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeyBox-WorkItem: 04a7948457d64d448cee6cd2e854d8de CodeyBox-Agent: antigravity/gemini-3.8-flash-high CodeyBox-Prompt-Revision: 1 CodeyBox-Fallbacks: antigravity→devin (×2 Agent antigravity rate-limited by provider (transient rate limit; retrying after backoff): agent exited 1) Co-Authored-By: CodeyBox --- CodeyBox.slnx | 1 + docs/concepts/agents.md | 3 + docs/reference/agent-quirks.md | 84 ++++ .../CodeyBox.Agents.Unreal.csproj | 19 + .../UnrealAgentRunner.cs | 466 ++++++++++++++++++ .../UnrealCostExtractor.cs | 118 +++++ .../UnrealInVmSmokeProbe.cs | 41 ++ .../UnrealKnownModels.cs | 67 +++ .../UnrealModelListProbe.cs | 16 + .../UnrealQuotaFailureDetector.cs | 93 ++++ .../UnrealSmokeProbe.cs | 56 +++ .../UnrealStreamParser.cs | 226 +++++++++ .../UnrealTerminalDiagnoser.cs | 103 ++++ src/CodeyBox.Agents/CliAgentRunnerBase.cs | 44 +- src/CodeyBox.Api/AgentClassesConfigBuilder.cs | 3 + src/CodeyBox.Api/CodeyBox.Api.csproj | 1 + src/CodeyBox.Api/Program.cs | 44 ++ src/CodeyBox.Api/agent-pricing-defaults.json | 9 +- src/CodeyBox.Api/appsettings.json | 6 +- src/CodeyBox.Core/AgentKind.cs | 1 + tests/CodeyBox.Tests/CodeyBox.Tests.csproj | 1 + .../CodeyBox.Tests/UnrealAgentRunnerTests.cs | 347 +++++++++++++ .../UnrealCostExtractorTests.cs | 59 +++ tests/CodeyBox.Tests/UnrealProbeTests.cs | 153 ++++++ .../UnrealQuotaFailureDetectorTests.cs | 71 +++ .../CodeyBox.Tests/UnrealStreamParserTests.cs | 171 +++++++ .../UnrealTerminalDiagnoserTests.cs | 53 ++ 27 files changed, 2250 insertions(+), 6 deletions(-) create mode 100644 src/CodeyBox.Agents.Unreal/CodeyBox.Agents.Unreal.csproj create mode 100644 src/CodeyBox.Agents.Unreal/UnrealAgentRunner.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealCostExtractor.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealInVmSmokeProbe.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealKnownModels.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealModelListProbe.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealQuotaFailureDetector.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealSmokeProbe.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealStreamParser.cs create mode 100644 src/CodeyBox.Agents.Unreal/UnrealTerminalDiagnoser.cs create mode 100644 tests/CodeyBox.Tests/UnrealAgentRunnerTests.cs create mode 100644 tests/CodeyBox.Tests/UnrealCostExtractorTests.cs create mode 100644 tests/CodeyBox.Tests/UnrealProbeTests.cs create mode 100644 tests/CodeyBox.Tests/UnrealQuotaFailureDetectorTests.cs create mode 100644 tests/CodeyBox.Tests/UnrealStreamParserTests.cs create mode 100644 tests/CodeyBox.Tests/UnrealTerminalDiagnoserTests.cs diff --git a/CodeyBox.slnx b/CodeyBox.slnx index 2619afa8f..02d0643b2 100644 --- a/CodeyBox.slnx +++ b/CodeyBox.slnx @@ -26,6 +26,7 @@ + diff --git a/docs/concepts/agents.md b/docs/concepts/agents.md index ae024e9fe..62deeb655 100644 --- a/docs/concepts/agents.md +++ b/docs/concepts/agents.md @@ -33,6 +33,7 @@ tooling, not in the agent runner contract. | `cmd` | `cmd` | `OPENROUTER_API_KEY` (provider API key resolved through the seeded `~/.commandcode/providers.json` `$OPENROUTER_API_KEY` reference — the file never carries the raw key; the runner also seeds a non-credential `~/.commandcode/auth.json` placeholder for plan-less `--local-only` BYOK — see [Command Code quirks](../reference/agent-quirks.md#command-code-cmd)) | `CODEYBOX_CMD_API_KEY` | | `crush` | `crush` | `OPENROUTER_API_KEY` (provider API key read directly from the environment — no config file is seeded; every dispatch also carries `CRUSH_DISABLE_METRICS=1` — see [Crush quirks](../reference/agent-quirks.md#crush-crush)) | `CODEYBOX_CRUSH_API_KEY` | | `dotnet-opencode` | `dotnet-opencode` | `DOTNETOPENCODE_CONFIG_JSON` (global `opencode.json` provider config, written to `~/.config/opencode/opencode.json`; supports `{env:VAR}` indirection — see [dotnet-opencode quirks](../reference/agent-quirks.md#dotnet-opencode-honadotnet-opencode)) | `CODEYBOX_DOTNETOPENCODE_CONFIG_JSON` | +| `unreal` | `unreal-agent-runner` | `OPENROUTER_API_KEY` (provider API key read directly from environment — no guest config file; other providers use `OPENAI_API_KEY`, `FIREWORKS_API_KEY`, or `UNREAL_HARNESS_LLM_API_KEY` — see [Unreal quirks](../reference/agent-quirks.md#unreal-agent-unreal-agent-runner)) | `CODEYBOX_UNREAL_API_KEY` (+ `CODEYBOX_UNREAL_PROVIDER`, `CODEYBOX_UNREAL_OPENAI_API_KEY`, `CODEYBOX_UNREAL_FIREWORKS_API_KEY`) | The sandbox-side env name is what the agent CLI reads. The host-side name is what the orchestrator looks up when building the credential bundle — for most @@ -80,6 +81,7 @@ the most common cause of fresh-class dispatch failures. | `cmd` | `npm install -g command-code@1.54.2` | Needs Node.js on the image. Version-pinned for reproducible bakes — re-verify the headless contract (`-p --output-format json`), the `--yolo` autonomy flag, and the `~/.commandcode/` file layout (see below) before bumping. The base image should also pre-seed `~/.commandcode/providers.json` (openrouter entry with the `$OPENROUTER_API_KEY` reference — never a raw key) and `~/.commandcode/auth.json` (non-credential presence placeholder) so ad-hoc runs work; the runner re-seeds both at dispatch (see [Command Code quirks](../reference/agent-quirks.md#command-code-cmd) and [sandbox baselines](../reference/sandbox-baselines.md)). | | `crush` | `npm install -g @charmland/crush@0.95.0` | Charm's agent CLI; needs Node.js on the image. Version-pinned for reproducible bakes — re-verify the headless contract (`run -q -m`, prompt on stdin, plain-text output, no `--yolo` on `run`) before bumping. See [Crush quirks](../reference/agent-quirks.md#crush-crush). | | `dotnet-opencode` | `dotnet tool install --global dotnet-opencode --prerelease` (after the exact .NET 11 preview SDK `11.0.100-preview.7.26381.103` — roll-forward is disabled — plus ripgrep on PATH) | Heaviest agent baseline: preview SDK + prerelease tool + `rg`. Pinned version `0.1.0-ci.20260905083303.33955573552.1`. See [dotnet-opencode quirks](../reference/agent-quirks.md#dotnet-opencode-honadotnet-opencode) and [`sandbox-baselines.md`](../reference/sandbox-baselines.md). | +| `unreal` | `curl -fsSL -o /tmp/unreal.tar.gz https://github.com/unreallabsai/unreal-agent/releases/download/v0.1.1/unreal-agent-runner_0.1.1_linux_amd64.tar.gz && echo "fad9cb9e6e6272a8d16fb4b90f985abb3132572413588f96622c6b1a82e34fcd /tmp/unreal.tar.gz" \| sha256sum -c - && tar -xzf /tmp/unreal.tar.gz -C /usr/local/bin unreal-agent-runner && chmod +x /usr/local/bin/unreal-agent-runner && rm /tmp/unreal.tar.gz` (for arm64: `0e61571dc9b83b429aaf9c89d8af372ff39a7ef50313fa2fd0ef15c6fa527d02`) | Pinned release v0.1.1 (commit `b7c9bf1c5c`), SHA256 checksum-verified. Installs `unreal-agent-runner` binary on PATH. Self-contained Go executable (no runtime required). See [Unreal quirks](../reference/agent-quirks.md#unreal-agent-unreal-agent-runner). | Verify each command against its upstream install docs at the time of baking — versions and install URLs change. Multipass and Incus keep independent bake @@ -206,6 +208,7 @@ credentials before they waste expensive compute. | `omp` | *(no network call — omp fronts ~60 providers, so no single endpoint validates the credential)* — verifies the bundle carries `OPENROUTER_API_KEY`; real auth check happens on first CLI call | `OPENROUTER_API_KEY` | | `continue` | *(no network call — Continue fronts hundreds of models, so no single endpoint validates the credential)* — verifies the bundle carries `OPENROUTER_API_KEY`; real auth check happens on first CLI call | `OPENROUTER_API_KEY` | | `dotnet-opencode` | *(no network call — provider-agnostic BYOK front with an interactive-only device login; any provider call spends real quota)* — verifies the bundle carries `DOTNETOPENCODE_CONFIG_JSON`; real auth check happens on first CLI call | `DOTNETOPENCODE_CONFIG_JSON` | +| `unreal` | *(no network call — multi-provider front; any provider call spends real quota)* — verifies the bundle carries `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, `FIREWORKS_API_KEY`, or `UNREAL_HARNESS_LLM_API_KEY` and rejects Codex subscription credentials; real auth check happens on first CLI call | `OPENROUTER_API_KEY` (or provider key) | Each probe sends the minimal possible request (`max_tokens=1`). A 2xx response means the credential is valid. 401/403 is classified as `"auth"` failure. diff --git a/docs/reference/agent-quirks.md b/docs/reference/agent-quirks.md index 35d6404c4..10be85ce9 100644 --- a/docs/reference/agent-quirks.md +++ b/docs/reference/agent-quirks.md @@ -2412,3 +2412,87 @@ unknown model id is configuration, not quota — mirroring kilo's `Model not found` exclusion), `Unknown flag` (dispatch construction, which the runner's pinned argv cannot produce), and quota/401 prose from reviewed repository content (patterns stay anchored to provider-shaped sentences). + +### Unreal Agent (`unreal`) + +**Install in the sandbox image** — add the install line to +`CodeyBox:MultipassExtraRuncmd` or `CodeyBox:Incus:ExtraRuncmd`, matching the +selected provider (verified against unreal-agent v0.1.1, commit `b7c9bf1c5c`, 2026-09-24): + +```sh +UNREAL_AGENT_VERSION=v0.1.1 +curl -fsSL -o /tmp/unreal-agent-runner "https://github.com/unreallabsai/unreal-agent/releases/download/${UNREAL_AGENT_VERSION}/unreal-agent-runner-linux-amd64" +printf '%s %s\n' "fad9cb9e6e6272a8d16fb4b90f985abb3132572413588f96622c6b1a82e34fcd" /tmp/unreal-agent-runner | sha256sum -c - +install -m 0755 /tmp/unreal-agent-runner /usr/local/bin/unreal-agent-runner +rm /tmp/unreal-agent-runner +``` + +Go-based autonomous agent runner ([repo](https://github.com/unreallabsai/unreal-agent), binary `unreal-agent-runner`). +The installer downloads the pinned release binary with its SHA256 checksum verified before execution +(`fad9cb9e6e6272a8d16fb4b90f985abb3132572413588f96622c6b1a82e34fcd` for `linux_amd64`, +`0e61571dc9b83b429aaf9c89d8af372ff39a7ef50313fa2fd0ef15c6fa527d02` for `linux_arm64`), +dropping `unreal-agent-runner` on PATH. + +**Non-interactive invocation (Trap 1: JSON on stdin).** The runner does NOT use `-p`, +`--prompt`, or `/dev/stdin`. The CLI expects a non-interactive JSON request payload +piped directly to standard input: + +```json +{"prompt":"","model":"","thinking_level":""} +``` + +This bypasses Linux's `MAX_ARG_STRLEN` (128 KiB per argv element) entirely, allowing +rework prompts and large contexts (> 128 KiB) to be delivered intact without truncation +or shell escape hazards. + +**Session and log isolation (Trap 2: Directories outside workspace).** By default, +`unreal-agent-runner` attempts to create session and log directories inside the current +working directory (`.unreal/logs`, `.unreal/sessions`). In CodeyBox, git working trees must +remain clean and unpolluted by runner operational artifacts. The runner always supplies: + +```sh +unreal-agent-runner -workspace -log-directory /tmp/codeybox-unreal//logs -session-directory /tmp/codeybox-unreal//sessions +``` + +This ensures session operations, scratchpad files, and internal operation logs remain +isolated outside the workspace worktree. + +**Workspace `.env` quarantine (Trap 3: Preventing configuration injection).** Unreal agent +automatically parses `.env` files located in the workspace directory. A malicious or +compromised repository containing a `.env` file could override: +- `SANDBOX_EGRESS_PROXY` (hijacking outbound agent network traffic to an untrusted proxy) +- `UNREAL_HARNESS_LLM_PROVIDER` or `UNREAL_HARNESS_LLM_BASE_URL` (tampering with model endpoints) +- Provider API keys and credentials + +To prevent configuration injection, `UnrealAgentRunner` probes for `.env` files in the workspace +prior to execution, moves them to a unique quarantine path (`.env.codeybox-quarantined-`), +and safely restores them in a `finally` block upon completion. If quarantine fails, the run is +aborted immediately to fail closed. + +**Account safety: Codex subscription rejection.** Unreal agent supports pay-per-API providers +(OpenAI API, OpenRouter, etc.). It does NOT support OpenAI Codex web subscription tokens or +session credentials. Attempting to use subscription credentials with raw API endpoints risks +account suspension or billing errors. `UnrealAgentRunner` and `UnrealSmokeProbe` actively inspect +credentials and reject subscription tokens (`codex_subscription` kind or Bearer JWT subscription shapes) +before any dispatch occurs. + +**Reasoning effort & model mapping.** `ReasoningMode` maps to `thinking_level` in the JSON request: +- `ReasoningMode.Low` -> `"low"` +- `ReasoningMode.Medium` -> `"medium"` +- `ReasoningMode.High` -> `"high"` +- `ReasoningMode.ExtraHigh` -> `"xhigh"` +- `ReasoningMode.Maximum` -> `"max"` + +Supported catalog models include `gpt-6-astra` and `openrouter/nvidia/nemotron-3.5-lightning:free`. +Custom model IDs are passed verbatim with startup warnings if unrecognized. + +**Exit codes & failure classification.** +- `0`: Success. +- `1`: Error (lifted to `TerminalDiagnostic` bounded to 500 chars via `UnrealTerminalDiagnoser`). +- `130`: Interrupted (SIGINT/SIGTERM), classified as `AgentFailureKind.Infrastructure` so the orchestrator retries or reschedules rather than treating it as an agent task failure. + +**Cost attribution & stream parsing.** `UnrealStreamParser` parses NDJSON session items emitted to stdout: +- `model_response`: Extracts assistant text, tool calls, and token usage (`Usage.InputTokens`, `Usage.OutputTokens`, `Usage.CachedInputTokens`). +- `tool_call_status`: Tracks tool execution results and byte sizes from shell operations. +- `type: "error"`: Extracts terminal error messages. +`UnrealCostExtractor` attributes costs based on token usage reported in `model_response` frames. diff --git a/src/CodeyBox.Agents.Unreal/CodeyBox.Agents.Unreal.csproj b/src/CodeyBox.Agents.Unreal/CodeyBox.Agents.Unreal.csproj new file mode 100644 index 000000000..ff8aae0c7 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/CodeyBox.Agents.Unreal.csproj @@ -0,0 +1,19 @@ + + + + + + + + + + net10.0 + enable + enable + + + + + + + diff --git a/src/CodeyBox.Agents.Unreal/UnrealAgentRunner.cs b/src/CodeyBox.Agents.Unreal/UnrealAgentRunner.cs new file mode 100644 index 000000000..41ac899e8 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealAgentRunner.cs @@ -0,0 +1,466 @@ +using System.Text.Json; +using CodeyBox.Agents; +using CodeyBox.Core; +using CodeyBox.Sandbox; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Drives the Unreal Labs unreal-agent CLI (binary unreal-agent-runner, +/// GitHub github.com/unreallabsai/unreal-agent, pinned release v0.1.1, +/// commit b7c9bf1c5c) in headless mode. +/// +/// Invocation Contract (Trap 1 avoidance): +/// unreal-agent-runner accepts flags -workspace, -log-directory, +/// and -session-directory. It has NO -p or --prompt flag; +/// passing /dev/stdin or a prompt on argv fails. Prompt, model, and thinking level +/// must travel strictly as a JSON request envelope on standard input: +/// {"prompt":..., "model":..., "thinking_level":...}. Linux's MAX_ARG_STRLEN +/// is 128 KiB per argv element, and stdin delivery supports prompts of arbitrary size. +/// +/// Log and Session Directories (Trap 2 avoidance): +/// By default, unreal-agent-runner creates logs inside <workspace>/logs +/// and sessions inside ~/.unreal-agent/sessions. If left inside the workspace, +/// logs pollute the repository under test and corrupt git diff calculations. The runner +/// explicitly directs logs to /home/ubuntu/.unreal-agent/logs and sessions to +/// /home/ubuntu/.unreal-agent/sessions, strictly outside the workspace. +/// .unreal-agent/sessions is declared in , +/// while logs are kept out of the scratchpad allowlist because of unbounded growth. +/// +/// Workspace .env Quarantine (Trap 3 avoidance): +/// At startup, unreal-agent-runner loads <workspace>/.env via loadDotEnv. +/// If SANDBOX_EGRESS_PROXY is present in that .env, it overwrites HTTPS_PROXY +/// in the process environment, allowing an untrusted repository to hijack egress traffic. +/// Furthermore, .env can override provider, model, base URL, and credentials. +/// The runner therefore quarantines any workspace .env file to a unique backup before +/// dispatch, and restores it in a finally block when execution completes. +/// +/// Authentication & Account Safety: +/// Supported pay-per-API providers include OpenRouter (OPENROUTER_API_KEY), +/// OpenAI (OPENAI_API_KEY), Fireworks (FIREWORKS_API_KEY), or +/// UNREAL_HARNESS_LLM_API_KEY. Host credential mapping routes +/// CODEYBOX_UNREAL_API_KEY to OPENROUTER_API_KEY. To protect against account +/// suspension or token theft, Codex subscription credentials (OPENAI_CODEX_ACCESS_TOKEN, +/// OPENAI_CODEX_AUTH_FILE, auth.json, or UNREAL_HARNESS_LLM_PROVIDER=openai-codex) +/// are strictly prohibited and fail fast. +/// +/// Exit Codes: +/// 0 indicates successful completion; 1 indicates error; 130 indicates external interruption +/// (SIGINT), which is classified as . +/// +public sealed class UnrealAgentRunner : CliAgentRunnerBase, IAgentDefaultModelProvider, IStructuredStreamAgentRunner +{ + private readonly AgentDefaultsSnapshot? _defaults; + + public UnrealAgentRunner() : this(defaults: null) { } + + public UnrealAgentRunner(AgentDefaultsSnapshot? defaults) + { + _defaults = defaults; + } + + public override AgentKind Kind => AgentKind.Unreal; + + /// Default executable name inside the sandbox. + public const string DefaultBinary = "unreal-agent-runner"; + + /// Path to the binary inside the sandbox. + public string Binary { get; init; } = DefaultBinary; + + /// Default log directory outside the workspace. + public const string DefaultLogDirectory = "/home/ubuntu/.unreal-agent/logs"; + + /// Log directory passed to -log-directory. + public string LogDirectory { get; init; } = DefaultLogDirectory; + + /// Default session directory outside the workspace. + public const string DefaultSessionDirectory = "/home/ubuntu/.unreal-agent/sessions"; + + /// Session directory passed to -session-directory. + public string SessionDirectory { get; init; } = DefaultSessionDirectory; + + public const string CredentialVariable = "OPENROUTER_API_KEY"; + public const string ProviderVariable = "UNREAL_HARNESS_LLM_PROVIDER"; + + public const string MissingCredentialMarker = + "no Unreal credential configured (set host CODEYBOX_UNREAL_API_KEY)"; + + public const string ProhibitedCodexCredentialMarker = + "Codex subscription credentials are prohibited for account-safety. Use pay-per-API credentials (e.g. OPENROUTER_API_KEY, OPENAI_API_KEY, or FIREWORKS_API_KEY)."; + + public const int SigintExitCode = 130; + + protected override IReadOnlyList ScratchpadHomeDirectories => [".unreal-agent/sessions"]; + + public string? DefaultModelId => _defaults?.GetDefault(Kind.Value); + + private static readonly HashSet ValidThinkingLevels = new(StringComparer.OrdinalIgnoreCase) + { + "low", "medium", "high", "xhigh", "max" + }; + + public Task SupportsStructuredStreamAsync(ISandbox sandbox, CancellationToken ct = default) + { + // unreal-agent-runner natively emits structured JSONL session events to stdout + return Task.FromResult(true); + } + + public override async Task RunAsync( + ISandbox sandbox, + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null, + CancellationToken ct = default, + Action? stdoutChunkCallback = null, + bool captureStructuredStream = false) + { + if (IsCodexSubscriptionCredential(credential)) + { + return new AgentResult( + Success: false, + Summary: ProhibitedCodexCredentialMarker, + Stdout: null, + Stderr: ProhibitedCodexCredentialMarker); + } + + var quarantine = await QuarantineDotEnvAsync(sandbox, workingDirectory, ct).ConfigureAwait(false); + if (quarantine.Failure is { } quarantineFailure) + return quarantineFailure; + + AgentResult result; + string? restoreNote; + try + { + result = await base.RunAsync( + sandbox, + workingDirectory, + prompt, + credential, + modelId, + reasoningMode, + ct, + stdoutChunkCallback, + captureStructuredStream).ConfigureAwait(false); + } + finally + { + restoreNote = await RestoreDotEnvAsync(sandbox, quarantine.Quarantined, ct).ConfigureAwait(false); + } + + return WithTerminalDiagnostic(WithRestoreNote(result, restoreNote)); + } + + public override async Task RunResumedAsync( + ISandbox sandbox, + string workingDirectory, + string prompt, + AgentCredential? credential, + AgentResumeContext resume, + string? modelId = null, + string? reasoningMode = null, + CancellationToken ct = default, + Action? stdoutChunkCallback = null) + { + if (IsCodexSubscriptionCredential(credential)) + { + return new AgentResult( + Success: false, + Summary: ProhibitedCodexCredentialMarker, + Stdout: null, + Stderr: ProhibitedCodexCredentialMarker); + } + + var quarantine = await QuarantineDotEnvAsync(sandbox, workingDirectory, ct).ConfigureAwait(false); + if (quarantine.Failure is { } quarantineFailure) + return quarantineFailure; + + AgentResult result; + string? restoreNote; + try + { + result = await base.RunResumedAsync( + sandbox, + workingDirectory, + prompt, + credential, + resume, + modelId, + reasoningMode, + ct, + stdoutChunkCallback).ConfigureAwait(false); + } + finally + { + restoreNote = await RestoreDotEnvAsync(sandbox, quarantine.Quarantined, ct).ConfigureAwait(false); + } + + return WithTerminalDiagnostic(WithRestoreNote(result, restoreNote)); + } + + public AgentFailureClassification ClassifyFailure(AgentResult result) + { + if (!result.Success && AgentSuspendResilience.ParseAgentExitCode(result.Summary) is { } exitCode) + { + if (exitCode == SigintExitCode) + { + return new AgentFailureClassification( + AgentFailureKind.Infrastructure, + Reason: "unreal-agent run interrupted by SIGINT (exit 130)"); + } + } + + if (result.ExecutionUnavailable) + { + return new AgentFailureClassification( + AgentFailureKind.Infrastructure, + Reason: "sandbox execution was unavailable"); + } + + if (AgentFailureClassifier.DetectAuthRequired(Kind, result.Stderr, result.Stdout) is { } authRequired) + return authRequired.Classification; + + if (result.Success) + return new AgentFailureClassification(AgentFailureKind.Normal); + + return AgentFailureClassifier.Classify(Kind, result.Stderr, result.Stdout, result.Summary); + } + + protected override AgentInvocation BuildInvocation( + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId, + string? reasoningMode, + bool captureStructuredStream) + { + var effectiveModel = !string.IsNullOrEmpty(modelId) ? modelId : DefaultModelId; + var effectiveWorkspace = !string.IsNullOrWhiteSpace(workingDirectory) ? workingDirectory : "."; + + var argv = new List + { + Binary, + "-workspace", + effectiveWorkspace, + "-log-directory", + LogDirectory, + "-session-directory", + SessionDirectory + }; + + var extraEnv = BuildProviderEnvironment(credential); + var stdin = FormatRequestJson(prompt, effectiveModel, reasoningMode); + + return new AgentInvocation( + argv, + ExtraEnvironment: extraEnv.Count == 0 ? null : extraEnv, + Stdin: stdin); + } + + protected override AgentInvocation BuildInvocation( + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null, + bool captureStructuredStream = false) + => BuildInvocation( + workingDirectory: string.Empty, + prompt, + credential, + modelId, + reasoningMode, + captureStructuredStream); + + protected override AgentInvocation BuildTextOnlyInvocation( + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null) + => BuildInvocation( + workingDirectory: string.Empty, + prompt, + credential, + modelId, + reasoningMode, + captureStructuredStream: false); + + internal static string FormatRequestJson(string prompt, string? model, string? reasoningMode) + { + var payload = new Dictionary + { + ["prompt"] = prompt + }; + + if (!string.IsNullOrWhiteSpace(model)) + { + payload["model"] = model; + } + + if (!string.IsNullOrWhiteSpace(reasoningMode) && ValidThinkingLevels.Contains(reasoningMode.Trim())) + { + payload["thinking_level"] = reasoningMode.Trim().ToLowerInvariant(); + } + + return JsonSerializer.Serialize(payload) + "\n"; + } + + internal static Dictionary BuildProviderEnvironment(AgentCredential? credential) + { + var env = new Dictionary(StringComparer.Ordinal); + if (credential is not null) + { + foreach (var (k, v) in credential.EnvironmentVariables) + { + env[k] = v; + } + } + + if (!env.ContainsKey(ProviderVariable)) + { + if (env.ContainsKey("OPENROUTER_API_KEY")) + env[ProviderVariable] = "openrouter"; + else if (env.ContainsKey("FIREWORKS_API_KEY")) + env[ProviderVariable] = "fireworks"; + else if (env.ContainsKey("OPENAI_API_KEY")) + env[ProviderVariable] = "openai"; + } + + return env; + } + + internal static bool IsCodexSubscriptionCredential(AgentCredential? credential) + { + if (credential is null) + return false; + + if (credential.EnvironmentVariables.ContainsKey("OPENAI_CODEX_ACCESS_TOKEN") + || credential.EnvironmentVariables.ContainsKey("OPENAI_CODEX_AUTH_FILE")) + { + return true; + } + + if (credential.EnvironmentVariables.TryGetValue(ProviderVariable, out var provider) + && string.Equals(provider, "openai-codex", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + foreach (var key in credential.Files.Keys) + { + if (key.EndsWith("auth.json", StringComparison.OrdinalIgnoreCase) + || key.Contains("codex", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + internal static async Task QuarantineDotEnvAsync( + ISandbox sandbox, + string workingDirectory, + CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(workingDirectory)) + return new DotEnvQuarantineOutcome(null, null); + + var root = workingDirectory.TrimEnd('/'); + var envFile = $"{root}/.env"; + + var probe = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["test", "-f", envFile], + }, ct).ConfigureAwait(false); + + if (!probe.Success) + return new DotEnvQuarantineOutcome(null, null); + + var backupSuffix = $".codeybox-quarantined-{Guid.NewGuid():N}"[..31]; + var backupFile = $"{envFile}{backupSuffix}"; + + var move = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["mv", "--", envFile, backupFile], + }, ct).ConfigureAwait(false); + + if (!move.Success) + { + return new DotEnvQuarantineOutcome( + null, + new AgentResult( + Success: false, + Summary: $"refusing Unreal dispatch: workspace .env is present but could not be quarantined (exit {move.ExitCode})", + Stdout: move.Stdout, + Stderr: move.Stderr)); + } + + return new DotEnvQuarantineOutcome(new QuarantinedDotEnv(envFile, backupFile), null); + } + + internal static async Task RestoreDotEnvAsync( + ISandbox sandbox, + QuarantinedDotEnv? quarantined, + CancellationToken ct) + { + if (quarantined is null) + return null; + + try + { + var reappeared = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["test", "-e", quarantined.Original], + }, ct).ConfigureAwait(false); + + if (reappeared.Success) + { + return $"workspace {quarantined.Original} reappeared during run; backup left at {quarantined.Backup}"; + } + + var restore = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["mv", "--", quarantined.Backup, quarantined.Original], + }, ct).ConfigureAwait(false); + + if (!restore.Success) + { + return $"failed to restore {quarantined.Original} from {quarantined.Backup} (exit {restore.ExitCode})"; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return $"failed to restore {quarantined.Original}: {ex.GetType().Name}"; + } + + return null; + } + + private static AgentResult WithTerminalDiagnostic(AgentResult result) + { + if (result.Success || result.TerminalDiagnostic is not null) + return result; + + var diag = UnrealTerminalDiagnoser.TryExtractTerminalError(result.Stdout, result.Stderr); + return diag is null ? result : result with { TerminalDiagnostic = diag }; + } + + private static AgentResult WithRestoreNote(AgentResult result, string? restoreNote) + { + if (string.IsNullOrEmpty(restoreNote)) + return result; + + var note = $".env restore incomplete: {restoreNote}"; + return result with + { + Stderr = string.IsNullOrEmpty(result.Stderr) ? note : $"{result.Stderr}\n{note}", + }; + } +} + +internal sealed record QuarantinedDotEnv(string Original, string Backup); + +internal sealed record DotEnvQuarantineOutcome( + QuarantinedDotEnv? Quarantined, + AgentResult? Failure); diff --git a/src/CodeyBox.Agents.Unreal/UnrealCostExtractor.cs b/src/CodeyBox.Agents.Unreal/UnrealCostExtractor.cs new file mode 100644 index 000000000..74ddd9a64 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealCostExtractor.cs @@ -0,0 +1,118 @@ +using System.Text.Json; +using CodeyBox.Agents; +using CodeyBox.Core; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Token usage extractor for Unreal Labs' unreal-agent structured output. +/// Scans session JSONL events for model_response frames carrying Data.Response.Usage. +/// +public sealed class UnrealCostExtractor : IAgentCostExtractor +{ + public AgentKind Kind => AgentKind.Unreal; + + public ModelRateConfig? DefaultPricing => null; + + public AgentCostSnapshot? TryExtract(string? agentStdout, string? agentStderr) + { + try + { + var fromStdout = ScanStream(agentStdout); + var fromStderr = ScanStream(agentStderr); + if (fromStdout is null) return fromStderr; + if (fromStderr is null) return fromStdout; + var stdoutTotal = fromStdout.InputTokens + fromStdout.CachedInputTokens + fromStdout.OutputTokens; + var stderrTotal = fromStderr.InputTokens + fromStderr.CachedInputTokens + fromStderr.OutputTokens; + return stderrTotal > stdoutTotal ? fromStderr : fromStdout; + } + catch + { + // Contract: implementations must never throw. + return null; + } + } + + private static AgentCostSnapshot? ScanStream(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + return null; + + var input = 0; + var output = 0; + var cached = 0; + var sawUsage = false; + + var reader = new StringReader(text); + while (reader.ReadLine() is { } rawLine) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line[0] != '{') + continue; + + try + { + using var doc = JsonDocument.Parse(line); + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object) + continue; + + // Unreal session item format: {"Kind":"model_response","Data":{"Response":{"Usage":{...}}}} + if (root.TryGetProperty("Kind", out var kindProp) + && string.Equals(kindProp.GetString(), "model_response", StringComparison.OrdinalIgnoreCase) + && root.TryGetProperty("Data", out var dataProp) + && dataProp.ValueKind == JsonValueKind.Object + && dataProp.TryGetProperty("Response", out var respProp) + && respProp.ValueKind == JsonValueKind.Object + && respProp.TryGetProperty("Usage", out var usageProp) + && usageProp.ValueKind == JsonValueKind.Object) + { + if (TryReadTokens(usageProp, out var inTokens, out var outTokens, out var cachedTokens)) + { + input = inTokens; + output = outTokens; + cached = cachedTokens; + sawUsage = true; + } + } + } + catch (JsonException) + { + // Non-JSON or broken line, continue scanning + } + } + + if (!sawUsage || (input == 0 && output == 0 && cached == 0)) + return null; + + return new AgentCostSnapshot( + InputTokens: input, + CachedInputTokens: cached, + OutputTokens: output, + ModelId: null); + } + + private static bool TryReadTokens(JsonElement usage, out int input, out int output, out int cached) + { + input = 0; + output = 0; + cached = 0; + + if (usage.TryGetProperty("InputTokens", out var inProp) && inProp.TryGetInt32(out var inVal)) + input = inVal; + else if (usage.TryGetProperty("input_tokens", out var inSnake) && inSnake.TryGetInt32(out var inSnakeVal)) + input = inSnakeVal; + + if (usage.TryGetProperty("OutputTokens", out var outProp) && outProp.TryGetInt32(out var outVal)) + output = outVal; + else if (usage.TryGetProperty("output_tokens", out var outSnake) && outSnake.TryGetInt32(out var outSnakeVal)) + output = outSnakeVal; + + if (usage.TryGetProperty("CachedInputTokens", out var cProp) && cProp.TryGetInt32(out var cVal)) + cached = cVal; + else if (usage.TryGetProperty("cached_input_tokens", out var cSnake) && cSnake.TryGetInt32(out var cSnakeVal)) + cached = cSnakeVal; + + return input > 0 || output > 0 || cached > 0; + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealInVmSmokeProbe.cs b/src/CodeyBox.Agents.Unreal/UnrealInVmSmokeProbe.cs new file mode 100644 index 000000000..a13d4b19e --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealInVmSmokeProbe.cs @@ -0,0 +1,41 @@ +using CodeyBox.Core; + +namespace CodeyBox.Agents.Unreal; + +/// +/// In-VM smoke check for Unreal Labs' unreal-agent: +/// +/// unreal-agent-runner -h — binary present and runnable on PATH. +/// When credentials are present, a minimal stdin prompt dispatch step +/// exercising the JSON-on-stdin request channel without polluting any repo workspace. +/// +/// +public sealed class UnrealInVmSmokeProbe : IInVmSmokeProbe +{ + public AgentKind Kind => AgentKind.Unreal; + + public IReadOnlyList BuildSteps(AgentCredential? credential) + { + var steps = new List + { + new( + [UnrealAgentRunner.DefaultBinary, "-h"], + FailureHint: "unreal-agent-runner binary not runnable on sandbox PATH") + }; + + if (credential is not null) + { + steps.Add(new( + [ + UnrealAgentRunner.DefaultBinary, + "-workspace", "/tmp", + "-log-directory", "/tmp", + "-session-directory", "/tmp" + ], + Stdin: UnrealAgentRunner.FormatRequestJson("ping", model: null, reasoningMode: null), + FailureHint: "unreal-agent-runner prompt dispatch failed")); + } + + return steps; + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealKnownModels.cs b/src/CodeyBox.Agents.Unreal/UnrealKnownModels.cs new file mode 100644 index 000000000..614964452 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealKnownModels.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.Logging; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Seed list of Unreal model IDs driving the warn-only config validator and the +/// . +/// +/// Unreal Labs' unreal-agent harness supports multiple backend providers +/// (OpenAI, OpenRouter, Fireworks, Ollama). The model can be specified either via +/// the request's JSON model field or via UNREAL_HARNESS_LLM_MODEL. +/// The upstream default model is gpt-6-astra on OpenAI; on OpenRouter, +/// qualified model IDs such as openrouter/nvidia/nemotron-3.5-lightning:free +/// or nvidia/nemotron-3.5-lightning:free are used. Unknown model IDs are +/// not rejected — validation only logs a warning so operators catch typos early +/// (mirroring Crush and Continue). +/// +public static class UnrealKnownModels +{ + public const string DefaultModel = "gpt-6-astra"; + public const string FreeModel = "openrouter/nvidia/nemotron-3.5-lightning:free"; + + /// + /// Curated seed of Unreal-accepted model IDs. + /// + public static readonly IReadOnlyList All = + [ + DefaultModel, + FreeModel, + "nvidia/nemotron-3.5-lightning:free", + "gpt-5.5", + "openai/gpt-5.5", + ]; + + public static bool IsKnown(string? modelId) + { + if (string.IsNullOrWhiteSpace(modelId)) + return false; + foreach (var m in All) + { + if (string.Equals(m, modelId, StringComparison.OrdinalIgnoreCase)) + return true; + } + return false; + } + + /// + /// Logs a warning when the operator-configured + /// for an Unreal member is not in . + /// + public static string? ValidateModelIdAgainstProviderList( + string classId, string? modelId, ILogger log) + { + if (string.IsNullOrWhiteSpace(modelId)) + return null; + + if (IsKnown(modelId)) + return null; + + var warning = + $"AgentClass '{classId}': Unreal model '{modelId}' is not in the curated known-models seed " + + $"({string.Join(", ", All)}). If this is a valid provider model ID, dispatch may succeed; " + + "otherwise double-check the spelling."; + log.LogWarning("{Warning}", warning); + return warning; + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealModelListProbe.cs b/src/CodeyBox.Agents.Unreal/UnrealModelListProbe.cs new file mode 100644 index 000000000..f859a1619 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealModelListProbe.cs @@ -0,0 +1,16 @@ +using CodeyBox.Core; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Model-list probe for Unreal. Returns the curated seed. +/// Model discovery in unreal-agent is provider-dependent, so a static seed is served for +/// validation warnings without gating dispatch. +/// +public sealed class UnrealModelListProbe : IAgentModelListProbe +{ + public AgentKind Kind => AgentKind.Unreal; + + public Task GetModelListAsync(CancellationToken ct) + => Task.FromResult(AgentModelListResult.Success(UnrealKnownModels.All)); +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealQuotaFailureDetector.cs b/src/CodeyBox.Agents.Unreal/UnrealQuotaFailureDetector.cs new file mode 100644 index 000000000..6d974ffaa --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealQuotaFailureDetector.cs @@ -0,0 +1,93 @@ +using CodeyBox.Agents; +using CodeyBox.Core; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Recognises quota, rate-limit, and auth failures emitted by the Unreal CLI runner. +/// Scans both stdout (which carries structured error frames) and stderr. +/// +public sealed class UnrealQuotaFailureDetector : IAgentQuotaFailureDetector +{ + public AgentKind Kind => AgentKind.Unreal; + + public static readonly IReadOnlyList DefaultPatterns = + [ + // Shared provider rate-limit patterns (transient throughput refusals) + .. SharedRateLimitPatterns.ProviderRateLimitPatterns, + + // Spend-limit / quota exhaustion + new("Key limit exceeded", QuotaFailureKind.LimitReached), + new("insufficient_quota", QuotaFailureKind.LimitReached), + new("insufficient credits", QuotaFailureKind.LimitReached), + new("credit balance too low", QuotaFailureKind.LimitReached), + new("quota exceeded", QuotaFailureKind.LimitReached), + new("quota exhausted", QuotaFailureKind.LimitReached), + new("usage limit reached", QuotaFailureKind.LimitReached), + new("RESOURCE_EXHAUSTED", QuotaFailureKind.LimitReached), + new("402 Payment Required", QuotaFailureKind.LimitReached), + new("HTTP 402", QuotaFailureKind.LimitReached), + + // Auth / credential failures + new("must be set", QuotaFailureKind.Unauthorized), + new("UNREAL_HARNESS_LLM_API_KEY", QuotaFailureKind.Unauthorized), + new("unsupported provider", QuotaFailureKind.Unauthorized), + new("API key is invalid", QuotaFailureKind.Unauthorized), + new("invalid api key", QuotaFailureKind.Unauthorized), + new("invalid_api_key", QuotaFailureKind.Unauthorized), + new("incorrect api key", QuotaFailureKind.Unauthorized), + new("authentication_error", QuotaFailureKind.Unauthorized), + new("Authentication failed", QuotaFailureKind.Unauthorized), + new("401 Unauthorized", QuotaFailureKind.Unauthorized), + new("API Error: 401", QuotaFailureKind.Unauthorized), + ]; + + private readonly IReadOnlyList _patterns; + + public UnrealQuotaFailureDetector() : this(additionalPatterns: null) { } + + public UnrealQuotaFailureDetector(IEnumerable? additionalPatterns) + { + if (additionalPatterns is null) + { + _patterns = DefaultPatterns; + return; + } + + var extras = additionalPatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)).ToArray(); + if (extras.Length == 0) + { + _patterns = DefaultPatterns; + return; + } + + var combined = new List(DefaultPatterns.Count + extras.Length); + combined.AddRange(DefaultPatterns); + combined.AddRange(extras); + _patterns = combined; + } + + public QuotaDetection? Detect(string? stderr, string? stdout) + { + if (string.IsNullOrEmpty(stderr) && string.IsNullOrEmpty(stdout)) + return null; + + foreach (var entry in _patterns) + { + var inStderr = !string.IsNullOrEmpty(stderr) && stderr.Contains(entry.Pattern, StringComparison.OrdinalIgnoreCase); + var inStdout = !string.IsNullOrEmpty(stdout) && stdout.Contains(entry.Pattern, StringComparison.OrdinalIgnoreCase); + if (!inStderr && !inStdout) + continue; + + var resetSources = new List(2); + if (!string.IsNullOrEmpty(stderr)) resetSources.Add(stderr); + if (!string.IsNullOrEmpty(stdout)) resetSources.Add(stdout); + return new QuotaDetection( + entry.Kind, + QuotaResetParser.TryParseResetAt(resetSources) + ?? QuotaResetParser.TryParseRetryAfterHeader(resetSources)); + } + + return null; + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealSmokeProbe.cs b/src/CodeyBox.Agents.Unreal/UnrealSmokeProbe.cs new file mode 100644 index 000000000..9cc2da01f --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealSmokeProbe.cs @@ -0,0 +1,56 @@ +using CodeyBox.Core; +using Microsoft.Extensions.Logging; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Credential viability check for Unreal Labs' unreal-agent. +/// Validates that a pay-per-API key is available (OPENROUTER_API_KEY, +/// OPENAI_API_KEY, FIREWORKS_API_KEY, or UNREAL_HARNESS_LLM_API_KEY) +/// and verifies that Codex subscription credentials are NOT present. +/// +public sealed class UnrealSmokeProbe : IAgentSmokeProbe +{ + private readonly ILogger? _log; + + public UnrealSmokeProbe(ILogger? log = null) + { + _log = log; + } + + public AgentKind Kind => AgentKind.Unreal; + + public Task SmokeTestAsync(AgentCredential credential, CancellationToken ct) + { + if (UnrealAgentRunner.IsCodexSubscriptionCredential(credential)) + { + _log?.LogWarning("Unreal smoke probe rejected Codex subscription credentials"); + return Task.FromResult(new AgentSmokeResult( + Ok: false, + FailureReason: UnrealAgentRunner.ProhibitedCodexCredentialMarker, + Duration: TimeSpan.Zero, + Category: SmokeFailureCategory.Persistent)); + } + + var hasApiKey = credential.EnvironmentVariables.TryGetValue("OPENROUTER_API_KEY", out var routerKey) && !string.IsNullOrWhiteSpace(routerKey) + || credential.EnvironmentVariables.TryGetValue("OPENAI_API_KEY", out var openaiKey) && !string.IsNullOrWhiteSpace(openaiKey) + || credential.EnvironmentVariables.TryGetValue("FIREWORKS_API_KEY", out var fireworksKey) && !string.IsNullOrWhiteSpace(fireworksKey) + || credential.EnvironmentVariables.TryGetValue("UNREAL_HARNESS_LLM_API_KEY", out var harnessKey) && !string.IsNullOrWhiteSpace(harnessKey); + + if (!hasApiKey) + { + _log?.LogDebug("Unreal smoke probe found no usable API key in credential bundle"); + return Task.FromResult(new AgentSmokeResult( + Ok: false, + FailureReason: UnrealAgentRunner.MissingCredentialMarker, + Duration: TimeSpan.Zero, + Category: SmokeFailureCategory.Persistent)); + } + + return Task.FromResult(new AgentSmokeResult( + Ok: true, + FailureReason: null, + Duration: TimeSpan.Zero, + Category: SmokeFailureCategory.None)); + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealStreamParser.cs b/src/CodeyBox.Agents.Unreal/UnrealStreamParser.cs new file mode 100644 index 000000000..828d38cfb --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealStreamParser.cs @@ -0,0 +1,226 @@ +using System.Text.Json; +using CodeyBox.Agents; +using CodeyBox.Core; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Structured stream parser for Unreal Labs' unreal-agent session items. +/// +/// The CLI writes session items to stdout as JSONL (types: input, +/// turn, model_response, tool_call_status, fork). +/// Each item carries PascalCase properties: Sequence, RecordedAt, +/// Kind, and Data. On error, it also writes a terminal +/// {"type":"error","message":"..."} event to stdout. +/// +public sealed class UnrealStreamParser : FlexibleAgentStreamParser +{ + public UnrealStreamParser(AgentStreamParserOptions? options = null) + : base(AgentKind.Unreal, options) + { + } + + public override bool TryClaim(JsonElement line) => + IsUnrealSessionJsonEvent(line); + + internal static bool IsUnrealSessionJsonEvent(JsonElement line) + { + if (line.ValueKind != JsonValueKind.Object) + return false; + + // Distinctive Unreal session item schema: Sequence (number), Kind (string), RecordedAt (string), Data (object) + return line.TryGetProperty("Sequence", out var seq) && seq.ValueKind == JsonValueKind.Number + && line.TryGetProperty("Kind", out var kind) && kind.ValueKind == JsonValueKind.String + && line.TryGetProperty("RecordedAt", out var rec) && rec.ValueKind == JsonValueKind.String + && line.TryGetProperty("Data", out var data) && data.ValueKind == JsonValueKind.Object; + } + + protected override ParsedEvent ParseEvent(JsonElement root) + { + var type = FirstString(root, "type"); + if (string.Equals(type, CliAgentRunnerBase.StderrEnvelopeType, StringComparison.OrdinalIgnoreCase)) + return base.ParseEvent(root); + + // Terminal error event from unreal-agent-runner + if (string.Equals(type, "error", StringComparison.OrdinalIgnoreCase)) + { + var errorMessage = FirstString(root, "message") ?? string.Empty; + return new ParsedEvent( + EventType: "error", + Timestamp: null, + IsAssistant: false, + ToolStarts: [], + ToolResults: [], + InputTokens: null, + OutputTokens: null, + CachedInputTokens: null, + EstimatedUsd: null, + TotalDuration: null, + TimeToFirstToken: null, + FinalText: errorMessage, + IsRecognized: true, + StderrText: errorMessage); + } + + if (root.TryGetProperty("Kind", out var kindProp) && kindProp.ValueKind == JsonValueKind.String) + { + var kind = kindProp.GetString()!; + var timestamp = TryTimestamp(root, "RecordedAt"); + + if (string.Equals(kind, "model_response", StringComparison.OrdinalIgnoreCase) + && root.TryGetProperty("Data", out var data) + && data.ValueKind == JsonValueKind.Object + && data.TryGetProperty("Response", out var resp) + && resp.ValueKind == JsonValueKind.Object) + { + var starts = new List(); + string? finalText = null; + var isAssistant = false; + + if (resp.TryGetProperty("Output", out var outputArray) + && outputArray.ValueKind == JsonValueKind.Array) + { + foreach (var item in outputArray.EnumerateArray()) + { + var itemType = FirstString(item, "Type"); + if (string.Equals(itemType, "message", StringComparison.OrdinalIgnoreCase) + && item.TryGetProperty("Data", out var msgData) + && msgData.ValueKind == JsonValueKind.Object) + { + isAssistant = true; + if (msgData.TryGetProperty("Text", out var textProp) + && textProp.ValueKind == JsonValueKind.String) + { + finalText = textProp.GetString(); + } + } + else if (string.Equals(itemType, "tool_call", StringComparison.OrdinalIgnoreCase) + && item.TryGetProperty("Data", out var toolData) + && toolData.ValueKind == JsonValueKind.Object) + { + isAssistant = true; + var callId = FirstString(toolData, "CallID") ?? Guid.NewGuid().ToString("N"); + var name = FirstString(toolData, "Name") ?? "unknown"; + var args = FirstString(toolData, "Arguments") ?? string.Empty; + starts.Add(new ToolBuilder(callId, name, args, timestamp)); + } + } + } + + int? inputTokens = null; + int? outputTokens = null; + int? cachedTokens = null; + + if (resp.TryGetProperty("Usage", out var usage) && usage.ValueKind == JsonValueKind.Object) + { + if (usage.TryGetProperty("InputTokens", out var inProp) && inProp.TryGetInt32(out var inVal)) + inputTokens = inVal; + else if (usage.TryGetProperty("input_tokens", out var inSnake) && inSnake.TryGetInt32(out var inSnakeVal)) + inputTokens = inSnakeVal; + + if (usage.TryGetProperty("OutputTokens", out var outProp) && outProp.TryGetInt32(out var outVal)) + outputTokens = outVal; + else if (usage.TryGetProperty("output_tokens", out var outSnake) && outSnake.TryGetInt32(out var outSnakeVal)) + outputTokens = outSnakeVal; + + if (usage.TryGetProperty("CachedInputTokens", out var cProp) && cProp.TryGetInt32(out var cVal)) + cachedTokens = cVal; + else if (usage.TryGetProperty("cached_input_tokens", out var cSnake) && cSnake.TryGetInt32(out var cSnakeVal)) + cachedTokens = cSnakeVal; + } + + return new ParsedEvent( + EventType: kind, + Timestamp: timestamp, + IsAssistant: isAssistant, + ToolStarts: starts, + ToolResults: [], + InputTokens: inputTokens, + OutputTokens: outputTokens, + CachedInputTokens: cachedTokens, + EstimatedUsd: null, + TotalDuration: null, + TimeToFirstToken: null, + FinalText: finalText, + IsRecognized: true, + StderrText: null); + } + + if (string.Equals(kind, "tool_call_status", StringComparison.OrdinalIgnoreCase) + && root.TryGetProperty("Data", out var statusData) + && statusData.ValueKind == JsonValueKind.Object) + { + var callId = FirstString(statusData, "CallID") ?? "unknown"; + var results = new List(); + + var statusError = string.Empty; + if (statusData.TryGetProperty("Status", out var statusObj) + && statusObj.ValueKind == JsonValueKind.Object) + { + statusError = FirstString(statusObj, "Error") ?? string.Empty; + } + + if (statusData.TryGetProperty("Operations", out var ops) && ops.ValueKind == JsonValueKind.Array) + { + foreach (var op in ops.EnumerateArray()) + { + var opStatus = FirstString(op, "Status"); + if (string.Equals(opStatus, "completed", StringComparison.OrdinalIgnoreCase)) + { + var exitCode = 0; + long outputBytes = 0; + + if (op.TryGetProperty("State", out var state) && state.ValueKind == JsonValueKind.Object + && state.TryGetProperty("Result", out var res) && res.ValueKind == JsonValueKind.Object) + { + if (res.TryGetProperty("ExitCode", out var ec) && ec.TryGetInt32(out var exitVal)) + exitCode = exitVal; + if (res.TryGetProperty("OutSize", out var os) && os.TryGetInt64(out var outSizeVal)) + outputBytes += outSizeVal; + if (res.TryGetProperty("ErrSize", out var es) && es.TryGetInt64(out var errSizeVal)) + outputBytes += errSizeVal; + } + + var succeeded = string.IsNullOrEmpty(statusError) && exitCode == 0; + results.Add(new ToolResultBuilder(callId, succeeded, (int)Math.Min(int.MaxValue, outputBytes), timestamp, Duration: null)); + } + } + } + + return new ParsedEvent( + EventType: kind, + Timestamp: timestamp, + IsAssistant: false, + ToolStarts: [], + ToolResults: results, + InputTokens: null, + OutputTokens: null, + CachedInputTokens: null, + EstimatedUsd: null, + TotalDuration: null, + TimeToFirstToken: null, + FinalText: null, + IsRecognized: true, + StderrText: null); + } + + return new ParsedEvent( + EventType: kind, + Timestamp: timestamp, + IsAssistant: false, + ToolStarts: [], + ToolResults: [], + InputTokens: null, + OutputTokens: null, + CachedInputTokens: null, + EstimatedUsd: null, + TotalDuration: null, + TimeToFirstToken: null, + FinalText: null, + IsRecognized: true, + StderrText: null); + } + + return base.ParseEvent(root); + } +} diff --git a/src/CodeyBox.Agents.Unreal/UnrealTerminalDiagnoser.cs b/src/CodeyBox.Agents.Unreal/UnrealTerminalDiagnoser.cs new file mode 100644 index 000000000..51c6ce4f8 --- /dev/null +++ b/src/CodeyBox.Agents.Unreal/UnrealTerminalDiagnoser.cs @@ -0,0 +1,103 @@ +using System.Text.Json; + +namespace CodeyBox.Agents.Unreal; + +/// +/// Pure extraction of Unreal's terminal failure diagnostic from stdout JSON error events +/// and stderr output lines. +/// +/// On any error, unreal-agent-runner writes a structured error event +/// {"type":"error","message":"..."} to stdout and writes the prefixed error +/// message unreal-agent-runner: ... to stderr, then exits 1 (or 130 on SIGINT). +/// This diagnoser extracts the root cause so the orchestrator can classify failures +/// cleanly rather than reporting generic non-zero exits. +/// +public static class UnrealTerminalDiagnoser +{ + public const int MaxDiagnosticChars = 500; + public const string BinaryPrefix = "unreal-agent-runner:"; + + public static string? TryExtractTerminalError(string? stdout, string? stderr) + { + // 1. Try to extract from stdout structured JSON error frame + if (TryExtractFromStdout(stdout) is { } stdoutError) + return stdoutError; + + // 2. Try to extract from stderr lines + if (TryExtractFromStderr(stderr) is { } stderrError) + return stderrError; + + // 3. Scan stdout line-by-line for any prefixed or JSON error lines as fallback + if (TryExtractFromStderr(stdout) is { } fallbackError) + return fallbackError; + + return null; + } + + private static string? TryExtractFromStdout(string? stdout) + { + if (string.IsNullOrWhiteSpace(stdout)) + return null; + + var reader = new StringReader(stdout); + while (reader.ReadLine() is { } line) + { + var trimmed = line.Trim(); + if (trimmed.Length == 0 || trimmed[0] != '{') + continue; + + try + { + using var doc = JsonDocument.Parse(trimmed); + var root = doc.RootElement; + if (root.ValueKind == JsonValueKind.Object + && root.TryGetProperty("type", out var typeProp) + && typeProp.ValueKind == JsonValueKind.String + && string.Equals(typeProp.GetString(), "error", StringComparison.OrdinalIgnoreCase) + && root.TryGetProperty("message", out var msgProp) + && msgProp.ValueKind == JsonValueKind.String) + { + var msg = msgProp.GetString(); + if (!string.IsNullOrWhiteSpace(msg)) + return Truncate(msg.Trim()); + } + } + catch (JsonException) + { + // Non-JSON or malformed lines are skipped + } + } + + return null; + } + + private static string? TryExtractFromStderr(string? stderr) + { + if (string.IsNullOrWhiteSpace(stderr)) + return null; + + var reader = new StringReader(stderr); + while (reader.ReadLine() is { } line) + { + var trimmed = line.Trim(); + if (trimmed.Length == 0) + continue; + + if (trimmed.StartsWith(BinaryPrefix, StringComparison.OrdinalIgnoreCase)) + { + var msg = trimmed[BinaryPrefix.Length..].Trim(); + if (!string.IsNullOrEmpty(msg)) + return Truncate(msg); + } + } + + return null; + } + + private static string Truncate(string message) + { + if (message.Length <= MaxDiagnosticChars) + return message; + return message[..MaxDiagnosticChars]; + } +} diff --git a/src/CodeyBox.Agents/CliAgentRunnerBase.cs b/src/CodeyBox.Agents/CliAgentRunnerBase.cs index 17530ff6f..f5906e6a9 100644 --- a/src/CodeyBox.Agents/CliAgentRunnerBase.cs +++ b/src/CodeyBox.Agents/CliAgentRunnerBase.cs @@ -63,6 +63,19 @@ protected abstract AgentInvocation BuildInvocation( string? reasoningMode = null, bool captureStructuredStream = false); + /// + /// Build the argv to execute inside the sandbox for a given prompt and working directory. + /// Overridden by runners that require the working directory to configure CLI flags. + /// + protected virtual AgentInvocation BuildInvocation( + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null, + bool captureStructuredStream = false) + => BuildInvocation(prompt, credential, modelId, reasoningMode, captureStructuredStream); + /// /// CLI state paths under HOME whose contents are useful for graceful /// preemption. The default preempt hook captures only these allowlisted @@ -143,6 +156,20 @@ protected virtual AgentInvocation BuildResumeInvocation( bool captureStructuredStream = false) => BuildInvocation(prompt, credential, modelId, reasoningMode, captureStructuredStream); + /// + /// Build the invocation used after a checkpoint restore with a given working directory. + /// Overridden by runners that require the working directory to configure CLI flags. + /// + protected virtual AgentInvocation BuildResumeInvocation( + string workingDirectory, + string prompt, + AgentCredential? credential, + AgentResumeContext resume, + string? modelId = null, + string? reasoningMode = null, + bool captureStructuredStream = false) + => BuildResumeInvocation(prompt, credential, resume, modelId, reasoningMode, captureStructuredStream); + /// /// Build the invocation used to continue a crashed native CLI session in /// the same sandbox. Only subclasses that implement @@ -411,7 +438,7 @@ public virtual async Task RunAsync( if (preparation is not null) return preparation; - var invocation = BuildInvocation(prompt, credential, modelId, reasoningMode, captureStructuredStream); + var invocation = BuildInvocation(workingDirectory, prompt, credential, modelId, reasoningMode, captureStructuredStream); return await ExecuteWithSuspendResilienceAsync( sandbox, workingDirectory, @@ -496,6 +523,7 @@ protected async Task RunResumedCoreAsync( return PreserveNativeSessionId(preparation, resume.NativeSessionId); var invocation = BuildResumeInvocation( + workingDirectory, prompt, credential, resume, @@ -1057,6 +1085,18 @@ private void FlushLocked() string? reasoningMode = null) => null; + /// + /// Build argv for text-only sandbox calls with a given working directory. + /// Overridden by runners that require the working directory to configure CLI flags. + /// + protected virtual AgentInvocation? BuildTextOnlyInvocation( + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null) + => BuildTextOnlyInvocation(prompt, credential, modelId, reasoningMode); + /// /// Viability probe for subscription CLIs whose sandbox auth materialisation /// no-ops when the auth-json env var is absent (image-baked CLI auth). @@ -1110,7 +1150,7 @@ protected async Task ExecuteTextOnlyInSandboxAsync( if (preparation is not null) return new TextOnlyAgentResult(false, preparation.Summary, preparation.Stdout, preparation.Stderr); - var invocation = BuildTextOnlyInvocation(prompt, credential, modelId, reasoningMode); + var invocation = BuildTextOnlyInvocation(workingDirectory, prompt, credential, modelId, reasoningMode); if (invocation is null) { return new TextOnlyAgentResult( diff --git a/src/CodeyBox.Api/AgentClassesConfigBuilder.cs b/src/CodeyBox.Api/AgentClassesConfigBuilder.cs index 4d4cc05d8..0dbe9d480 100644 --- a/src/CodeyBox.Api/AgentClassesConfigBuilder.cs +++ b/src/CodeyBox.Api/AgentClassesConfigBuilder.cs @@ -17,6 +17,7 @@ using CodeyBox.Agents.Qwen; using CodeyBox.Agents.Vibe; using CodeyBox.Agents.DotNetOpencode; +using CodeyBox.Agents.Unreal; using CodeyBox.Core; using CodeyBox.Orchestrator; using Microsoft.Extensions.Logging; @@ -136,6 +137,8 @@ public static IReadOnlyList Build( DevinKnownModels.ValidateModelIdAgainstProviderList(classOpts.Id, m.ModelId, log); if (agentKind == AgentKind.DotNetOpencode) DotNetOpencodeKnownModels.ValidateModelIdAgainstProviderList(classOpts.Id, m.ModelId, log); + if (agentKind == AgentKind.Unreal) + UnrealKnownModels.ValidateModelIdAgainstProviderList(classOpts.Id, m.ModelId, log); // Capabilities are operator-declared tags. Normalise (trim + drop empties) // and de-duplicate case-insensitively so '"sensitive"' and '"Sensitive"' // don't both end up in the list. Tag values themselves are otherwise diff --git a/src/CodeyBox.Api/CodeyBox.Api.csproj b/src/CodeyBox.Api/CodeyBox.Api.csproj index 924cb2fe6..edd1a6e5d 100644 --- a/src/CodeyBox.Api/CodeyBox.Api.csproj +++ b/src/CodeyBox.Api/CodeyBox.Api.csproj @@ -30,6 +30,7 @@ + diff --git a/src/CodeyBox.Api/Program.cs b/src/CodeyBox.Api/Program.cs index 65a5df679..912a5b96e 100644 --- a/src/CodeyBox.Api/Program.cs +++ b/src/CodeyBox.Api/Program.cs @@ -30,6 +30,7 @@ using CodeyBox.Agents.Qwen; using CodeyBox.Agents.Vibe; using CodeyBox.Agents.DotNetOpencode; +using CodeyBox.Agents.Unreal; using CodeyBox.AdminSeed; using CodeyBox.Api; using CodeyBox.Api.Hubs; @@ -1660,6 +1661,15 @@ static string FormatBytes(long bytes) // docs/reference/agent-quirks.md. builder.Services.AddSingleton(sp => new DotNetOpencodeAgentRunner( sp.GetRequiredService())); +// Unreal: Unreal Labs' unreal-agent CLI (unreal-agent-runner, pinned release v0.1.1, +// commit b7c9bf1c5c). Transport delivers prompt, model, and thinking_level on stdin +// as a JSON request (NO -p, NO /dev/stdin). Log and session directories are placed +// strictly outside the workspace (/home/ubuntu/.unreal-agent/logs and sessions). +// The runner quarantines workspace .env before dispatch to neutralize egress proxy +// redirection and provider override attacks. See docs/concepts/agents.md and +// docs/reference/agent-quirks.md. +builder.Services.AddSingleton(sp => new UnrealAgentRunner( + sp.GetRequiredService())); // Seeded fake-agent run mode for the admin E2E/demo instance (see // docs/concepts/admin-e2e.md). Opt-in via CodeyBox:SeededFakeAgents:Enabled; // when disabled nothing here registers and production routing is untouched. @@ -2237,6 +2247,12 @@ static string FormatBytes(long bytes) // {env:VAR} indirection inside the JSON and add that provider's // variable as a second mapping row. new AgentCredentialMapping(AgentKind.DotNetOpencode, "CODEYBOX_DOTNETOPENCODE_CONFIG_JSON", "DOTNETOPENCODE_CONFIG_JSON"), + // Unreal: provider API-key auth read directly from environment. + // Shipped mapping routes host CODEYBOX_UNREAL_API_KEY to OPENROUTER_API_KEY. + new AgentCredentialMapping(AgentKind.Unreal, "CODEYBOX_UNREAL_API_KEY", "OPENROUTER_API_KEY"), + new AgentCredentialMapping(AgentKind.Unreal, "CODEYBOX_UNREAL_PROVIDER", "UNREAL_HARNESS_LLM_PROVIDER"), + new AgentCredentialMapping(AgentKind.Unreal, "CODEYBOX_UNREAL_OPENAI_API_KEY", "OPENAI_API_KEY"), + new AgentCredentialMapping(AgentKind.Unreal, "CODEYBOX_UNREAL_FIREWORKS_API_KEY", "FIREWORKS_API_KEY"), })); // Antigravity uses Sign-in-with-Google OAuth. The dedicated provider ships // the agy token bundle verbatim (refresh_token RETAINED) into the sandbox, @@ -2272,6 +2288,9 @@ static string FormatBytes(long bytes) new AgentCredentialMapping(AgentKind.CavemanCode, "OPENAI_API_KEY", "OPENAI_API_KEY"), new AgentCredentialMapping(AgentKind.CavemanCode, "GEMINI_API_KEY", "GEMINI_API_KEY"), new AgentCredentialMapping(AgentKind.CavemanCode, "OPENROUTER_API_KEY", "OPENROUTER_API_KEY"), + new AgentCredentialMapping(AgentKind.Unreal, "OPENROUTER_API_KEY", "OPENROUTER_API_KEY"), + new AgentCredentialMapping(AgentKind.Unreal, "OPENAI_API_KEY", "OPENAI_API_KEY"), + new AgentCredentialMapping(AgentKind.Unreal, "FIREWORKS_API_KEY", "FIREWORKS_API_KEY"), })); return new ChainedCredentialProvider( @@ -3136,6 +3155,12 @@ static IAgentQuotaProbe WireQuotaProbeTokenInvalidation( builder.Services.AddSingleton(sp => new DotNetOpencodeSmokeProbe( sp.GetRequiredService().CreateLogger())); +// Unreal: credential-presence check (OPENROUTER_API_KEY, OPENAI_API_KEY, +// FIREWORKS_API_KEY, or UNREAL_HARNESS_LLM_API_KEY in the bundle) and +// rejection of Codex subscription credentials for account-safety. +builder.Services.AddSingleton(sp => + new UnrealSmokeProbe( + sp.GetRequiredService().CreateLogger())); // --- In-VM smoke probes ------------------------------------------------------ // Registered as IEnumerable; InVmSmokeProber resolves by Kind. @@ -3165,6 +3190,7 @@ static IAgentQuotaProbe WireQuotaProbeTokenInvalidation( builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); // Startup guard (AC#1): bench any configured AgentClass member with no in-VM // probe (so a CLI-backed agent that would fail at first dispatch is routed past // at smoke time, not first dispatch). Agents on @@ -3342,6 +3368,7 @@ static IAgentQuotaProbe WireQuotaProbeTokenInvalidation( // DotNetOpencodeKnownModels seed is authoritative; operator-configured ids // absent from the seed surface as a startup warning, not a hard reject. builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddHostedService(); builder.Services.AddSingleton(sp => @@ -4417,6 +4444,7 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) [AgentKind.Cmd] = new CmdCostExtractor(), [AgentKind.Crush] = new CrushCostExtractor(), [AgentKind.Devin] = new DevinCostExtractor(), + [AgentKind.Unreal] = new UnrealCostExtractor(), }; // Warn once at startup for registered agents with no extractor. foreach (var kind in registry.Available) @@ -4555,6 +4583,7 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) // Crush work items to AgentKind.Crush rather than unknown (same as // aider/opencode/continue). builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Per-provider buffered-stdout tool-call counters. Used by the orchestrator @@ -4823,6 +4852,21 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) .ToArray(); return new CrushQuotaFailureDetector(extras); }); +builder.Services.AddSingleton(sp => +{ + // Unreal detector accepts operator-extensible patterns from + // CodeyBox:QuotaFailurePatterns:unreal, mirroring the crush hook above. + var cbOpts = sp.GetRequiredService>().Value; + var extras = cbOpts.QuotaFailurePatterns is null + ? null + : cbOpts.QuotaFailurePatterns + .Where(kvp => string.Equals(kvp.Key, AgentKind.Unreal.Value, StringComparison.OrdinalIgnoreCase)) + .SelectMany(kvp => kvp.Value ?? new List()) + .Where(p => !string.IsNullOrEmpty(p.Pattern)) + .Select(p => new QuotaFailurePattern(p.Pattern, p.Kind)) + .ToArray(); + return new UnrealQuotaFailureDetector(extras); +}); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/CodeyBox.Api/agent-pricing-defaults.json b/src/CodeyBox.Api/agent-pricing-defaults.json index 7912ec68a..d4043b70c 100644 --- a/src/CodeyBox.Api/agent-pricing-defaults.json +++ b/src/CodeyBox.Api/agent-pricing-defaults.json @@ -19,7 +19,8 @@ "continue": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)", "qwen": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)", "cmd": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)", - "crush": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)" + "crush": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)", + "unreal": "https://openrouter.ai/models (free-tier :free models bill $0; paid OpenRouter models bill OpenRouter list prices)" }, "notes": { "opencode": "OpenCode Go is subscription-priced ($12 per 5h usage budget on https://opencode.ai/docs/go). Each model entry uses one subscription-equivalent USD/M rate for input, cached input, and output: ($12 ÷ requests-per-5h) ÷ ((typical input + cached + output tokens per request) ÷ 1e6), using the request limits and typical token mix published on that page. Keys use opencode-go/. Cost attribution charges the stored non-cached input bucket and cached input bucket separately; public total-input reporting adds them.", @@ -42,7 +43,8 @@ "continue": "Continue is a multi-provider front; rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly for the seeded guest-config model id. The headless transport carries no machine-readable usage frame (only usage lives in the guest session files the extractor cannot reach), so the cost extractor reports unknown and per-run rows fall back to elapsed-time attribution; the bucket exists so a future usage-bearing build prices the shipped model correctly. Operators fronting paid OpenRouter models add that model's OpenRouter list prices here (or under CodeyBox:AgentPricing) keyed by the same model id form.", "qwen": "Qwen is a multi-provider front (native Qwen, OpenAI-, Anthropic-, and Gemini-compatible endpoints); rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly for the dispatched -m id (verified live: the result frame reports usage with the dispatch id verbatim in message.model, plus a per-model breakdown in stats.models). Keys are the provider-native ids qwen reports verbatim in message.model. Operators fronting paid models add that model's list prices here (or under CodeyBox:AgentPricing) keyed by the same id form.", "cmd": "Command Code is a multi-provider front; rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly keyed by the full dispatch id cmd echoes in model_request_start.model (verified live: the result line reports cost_usd 0). Operators fronting paid OpenRouter models add that model's OpenRouter list prices here (or under CodeyBox:AgentPricing) keyed by the qualified -m id form.", - "crush": "Crush is a multi-provider front (Charm); rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly for the dispatched -m id. The headless transport carries no machine-readable usage frame (plain-text replies only), so the cost extractor reports unknown and per-run rows fall back to elapsed-time attribution; the bucket exists so a future usage-bearing build prices the shipped model correctly. Operators fronting paid OpenRouter models add that model's OpenRouter list prices here (or under CodeyBox:AgentPricing) keyed by the qualified -m id form." + "crush": "Crush is a multi-provider front (Charm); rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly for the dispatched -m id. The headless transport carries no machine-readable usage frame (plain-text replies only), so the cost extractor reports unknown and per-run rows fall back to elapsed-time attribution; the bucket exists so a future usage-bearing build prices the shipped model correctly. Operators fronting paid OpenRouter models add that model's OpenRouter list prices here (or under CodeyBox:AgentPricing) keyed by the qualified -m id form.", + "unreal": "Unreal Agent is a multi-provider front (Unreal Labs); rates depend on the backing provider. The shipped AgentClasses member routes an OpenRouter free-tier model, which bills $0 — the bucket records that zero rate explicitly for the dispatched model id. Usage is extracted from session JSONL model_response frames. Operators fronting paid OpenRouter models add that model's OpenRouter list prices here (or under CodeyBox:AgentPricing) keyed by the qualified model id form." } }, "DefaultRates": { @@ -139,6 +141,9 @@ }, "crush": { "openrouter/nvidia/nemotron-3.5-lightning:free": { "inputPerMillion": 0.00, "cachedInputPerMillion": 0.00, "outputPerMillion": 0.00 } + }, + "unreal": { + "openrouter/nvidia/nemotron-3.5-lightning:free": { "inputPerMillion": 0.00, "cachedInputPerMillion": 0.00, "outputPerMillion": 0.00 } } } } diff --git a/src/CodeyBox.Api/appsettings.json b/src/CodeyBox.Api/appsettings.json index 88aaa05e6..fccc75245 100644 --- a/src/CodeyBox.Api/appsettings.json +++ b/src/CodeyBox.Api/appsettings.json @@ -111,7 +111,8 @@ "continue": "nvidia/nemotron-3.5-lightning:free", "qwen": "nvidia/nemotron-3.5-lightning:free", "cmd": "openrouter/nvidia/nemotron-3.5-lightning:free", - "crush": "openrouter/nvidia/nemotron-3.5-lightning:free" + "crush": "openrouter/nvidia/nemotron-3.5-lightning:free", + "unreal": "openrouter/nvidia/nemotron-3.5-lightning:free" }, "Goose": { "Provider": "openrouter", @@ -186,7 +187,8 @@ { "Agent": "continue", "Billing": "PayPerApi", "ModelId": "nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 }, { "Agent": "qwen", "Billing": "PayPerApi", "ModelId": "nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 }, { "Agent": "cmd", "Billing": "PayPerApi", "ModelId": "openrouter/nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 }, - { "Agent": "crush", "Billing": "PayPerApi", "ModelId": "openrouter/nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 } + { "Agent": "crush", "Billing": "PayPerApi", "ModelId": "openrouter/nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 }, + { "Agent": "unreal", "Billing": "PayPerApi", "ModelId": "openrouter/nvidia/nemotron-3.5-lightning:free", "QualityScore": 70 } ] } ], diff --git a/src/CodeyBox.Core/AgentKind.cs b/src/CodeyBox.Core/AgentKind.cs index 24af2cf87..ffd37dd07 100644 --- a/src/CodeyBox.Core/AgentKind.cs +++ b/src/CodeyBox.Core/AgentKind.cs @@ -30,6 +30,7 @@ public readonly record struct AgentKind(string Value) public static AgentKind Cmd { get; } = new("cmd"); public static AgentKind Crush { get; } = new("crush"); public static AgentKind Devin { get; } = new("devin"); + public static AgentKind Unreal { get; } = new("unreal"); public override string ToString() => Value; } diff --git a/tests/CodeyBox.Tests/CodeyBox.Tests.csproj b/tests/CodeyBox.Tests/CodeyBox.Tests.csproj index b68c28713..c01b0a847 100644 --- a/tests/CodeyBox.Tests/CodeyBox.Tests.csproj +++ b/tests/CodeyBox.Tests/CodeyBox.Tests.csproj @@ -109,6 +109,7 @@ + diff --git a/tests/CodeyBox.Tests/UnrealAgentRunnerTests.cs b/tests/CodeyBox.Tests/UnrealAgentRunnerTests.cs new file mode 100644 index 000000000..36ef83d5a --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealAgentRunnerTests.cs @@ -0,0 +1,347 @@ +using System.Text.Json; +using CodeyBox.Agents; +using CodeyBox.Agents.Unreal; +using CodeyBox.Core; +using CodeyBox.Sandbox; + +namespace CodeyBox.Tests; + +/// +/// Tests for : +/// - Transport contract (Trap 1): NO -p, NO /dev/stdin, prompt on stdin as JSON +/// - Directories (Trap 2): -log-directory and -session-directory outside workspace +/// - Workspace .env neutralisation (Trap 3): quarantine before dispatch, restore in finally +/// - Prompt > 128 KiB delivered intact on stdin +/// - ReasoningMode mapped to thinking_level; model mapped to model +/// - Codex subscription credentials rejected +/// - Exit code classification: 0 ok, 1 error, 130 infrastructure (SIGINT) +/// +public sealed class UnrealAgentRunnerTests +{ + private const string FreeModel = "openrouter/nvidia/nemotron-3.5-lightning:free"; + + private static UnrealAgentRunner Runner(AgentDefaultsSnapshot? defaults = null) => + new(defaults: defaults); + + private static UnrealAgentRunner RunnerWithDefault(string model = FreeModel) => + Runner(new AgentDefaultsSnapshot( + new Dictionary(StringComparer.OrdinalIgnoreCase) { ["unreal"] = model })); + + private static AgentCredential Cred(string key = "test-key") => + new(AgentKind.Unreal, + new Dictionary { ["OPENROUTER_API_KEY"] = key }, + new Dictionary()); + + private static SandboxExec UnrealExec(UnrealDispatchSandbox sandbox) => + Assert.Single(sandbox.Execs, e => e.Argv.Count > 0 && e.Argv[0] == UnrealAgentRunner.DefaultBinary); + + [Fact] + public void Kind_IsUnreal() + { + Assert.Equal(AgentKind.Unreal, new UnrealAgentRunner().Kind); + } + + [Fact] + public void AgentKind_Unreal_RoundTrips() + { + Assert.Equal(AgentKind.Unreal, new AgentKind("unreal")); + } + + [Fact] + public void ScratchpadHomeDirectories_IncludesSessions_ExcludesLogs() + { + // Unreal sessions live in ~/.unreal-agent/sessions which belongs in scratchpad. + // Logs are unbounded and captured in DB, so they must NOT be in scratchpad. + var runner = new UnrealAgentRunner(); + // Invoke protected ScratchpadHomeDirectories via reflection or subclass + var prop = typeof(UnrealAgentRunner).GetProperty("ScratchpadHomeDirectories", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public); + var dirs = (IReadOnlyList)prop!.GetValue(runner)!; + + Assert.Contains(".unreal-agent/sessions", dirs); + Assert.DoesNotContain(".unreal-agent/logs", dirs); + } + + [Fact] + public async Task RunAsync_Argv_SetsWorkspaceAndDirectoriesStrictlyOutsideWorkspace() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + const string workspace = "/sandbox/work/my-repo"; + + await runner.RunAsync(sandbox, workspace, "do the work", Cred()); + + var exec = UnrealExec(sandbox); + var argv = exec.Argv.ToList(); + + // Must invoke binary with -workspace, -log-directory, -session-directory + Assert.Equal(UnrealAgentRunner.DefaultBinary, argv[0]); + Assert.Contains("-workspace", argv); + var wsIndex = argv.IndexOf("-workspace"); + Assert.Equal(workspace, argv[wsIndex + 1]); + + Assert.Contains("-log-directory", argv); + var logIndex = argv.IndexOf("-log-directory"); + var logDir = argv[logIndex + 1]; + Assert.False(logDir.StartsWith(workspace, StringComparison.Ordinal), "log directory must be strictly outside workspace"); + + Assert.Contains("-session-directory", argv); + var sessionIndex = argv.IndexOf("-session-directory"); + var sessionDir = argv[sessionIndex + 1]; + Assert.False(sessionDir.StartsWith(workspace, StringComparison.Ordinal), "session directory must be strictly outside workspace"); + + // Trap 1: NO -p, NO --prompt, NO /dev/stdin + Assert.DoesNotContain("-p", argv); + Assert.DoesNotContain("--prompt", argv); + Assert.DoesNotContain("/dev/stdin", argv); + } + + [Fact] + public async Task RunAsync_Prompt_TravelsViaStdin_AsJsonRequest() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + const string prompt = "fix the bug in program.cs"; + + await runner.RunAsync(sandbox, "/work", prompt, Cred()); + + var exec = UnrealExec(sandbox); + Assert.NotNull(exec.Stdin); + + // Parse stdin as JSON + using var doc = JsonDocument.Parse(exec.Stdin); + var root = doc.RootElement; + Assert.True(root.TryGetProperty("prompt", out var promptProp)); + Assert.Equal(prompt, promptProp.GetString()); + + // Argv must NOT contain prompt text + Assert.DoesNotContain(exec.Argv, a => a.Contains("fix the bug", StringComparison.Ordinal)); + } + + [Fact] + public async Task RunAsync_LargePrompt_Over128KiB_DeliveredIntactOnStdin() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + // Generate prompt larger than Linux MAX_ARG_STRLEN (128 KiB) + var largePrompt = new string('A', 140 * 1024); + + await runner.RunAsync(sandbox, "/work", largePrompt, Cred()); + + var exec = UnrealExec(sandbox); + Assert.NotNull(exec.Stdin); + + using var doc = JsonDocument.Parse(exec.Stdin); + var root = doc.RootElement; + Assert.True(root.TryGetProperty("prompt", out var promptProp)); + Assert.Equal(largePrompt, promptProp.GetString()); + } + + [Fact] + public async Task RunAsync_ModelAndThinkingLevel_MappedInJson() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = Runner(); + + await runner.RunAsync(sandbox, "/work", "test", Cred(), modelId: "openrouter/anthropic/claude-3.5-sonnet", reasoningMode: "HIGH"); + + var exec = UnrealExec(sandbox); + using var doc = JsonDocument.Parse(exec.Stdin!); + var root = doc.RootElement; + + Assert.Equal("openrouter/anthropic/claude-3.5-sonnet", root.GetProperty("model").GetString()); + Assert.Equal("high", root.GetProperty("thinking_level").GetString()); + } + + [Fact] + public async Task RunAsync_InvalidThinkingLevel_IsOmitted() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + + await runner.RunAsync(sandbox, "/work", "test", Cred(), reasoningMode: "ultra-extreme-invalid"); + + var exec = UnrealExec(sandbox); + using var doc = JsonDocument.Parse(exec.Stdin!); + var root = doc.RootElement; + + Assert.False(root.TryGetProperty("thinking_level", out _)); + } + + [Fact] + public async Task RunAsync_ProviderInferredFromApiKey_WhenNotSet() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + + var openrouterCred = new AgentCredential(AgentKind.Unreal, + new Dictionary { ["OPENROUTER_API_KEY"] = "sk-or-test" }, + new Dictionary()); + await runner.RunAsync(sandbox, "/work", "test", openrouterCred); + + var exec = UnrealExec(sandbox); + Assert.NotNull(exec.ExtraEnvironment); + Assert.Equal("openrouter", exec.ExtraEnvironment["UNREAL_HARNESS_LLM_PROVIDER"]); + } + + [Fact] + public async Task RunAsync_CodexSubscriptionCredential_ProhibitedAndRejectedFast() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + + var codexEnvCred = new AgentCredential(AgentKind.Unreal, + new Dictionary + { + ["OPENROUTER_API_KEY"] = "key", + ["OPENAI_CODEX_ACCESS_TOKEN"] = "token123" + }, + new Dictionary()); + + var result = await runner.RunAsync(sandbox, "/work", "test", codexEnvCred); + + Assert.False(result.Success); + Assert.Equal(UnrealAgentRunner.ProhibitedCodexCredentialMarker, result.Summary); + Assert.Empty(sandbox.Execs); + } + + [Fact] + public async Task RunAsync_CodexAuthJsonFile_ProhibitedAndRejectedFast() + { + var sandbox = new UnrealDispatchSandbox(); + var runner = RunnerWithDefault(); + + var codexFileCred = new AgentCredential(AgentKind.Unreal, + new Dictionary { ["OPENROUTER_API_KEY"] = "key" }, + new Dictionary { ["~/.config/codex/auth.json"] = "{}" }); + + var result = await runner.RunAsync(sandbox, "/work", "test", codexFileCred); + + Assert.False(result.Success); + Assert.Equal(UnrealAgentRunner.ProhibitedCodexCredentialMarker, result.Summary); + Assert.Empty(sandbox.Execs); + } + + [Fact] + public async Task RunAsync_WorkspaceDotEnv_QuarantinedDuringDispatchAndRestored() + { + var sandbox = new UnrealDispatchSandbox(); + sandbox.Files.Add("/work/.env"); + var runner = RunnerWithDefault(); + + var result = await runner.RunAsync(sandbox, "/work", "test", Cred()); + + Assert.True(result.Success); + // .env was moved aside during execution and restored afterwards + Assert.Contains("/work/.env", sandbox.Files); + + // Verify mv operations: first to quarantine backup, then back to original + var moves = sandbox.Execs.Where(e => e.Argv.Count >= 2 && e.Argv[0] == "mv").ToList(); + Assert.Equal(2, moves.Count); + + // Move 1: /work/.env -> /work/.env.codeybox-quarantined-... + Assert.Equal("/work/.env", moves[0].Argv[2]); + Assert.StartsWith("/work/.env.codeybox-quarantined-", moves[0].Argv[3]); + + // Move 2: restore backup -> /work/.env + Assert.Equal(moves[0].Argv[3], moves[1].Argv[2]); + Assert.Equal("/work/.env", moves[1].Argv[3]); + } + + [Fact] + public async Task RunAsync_WorkspaceDotEnv_QuarantineFailure_FailsClosed() + { + var sandbox = new UnrealDispatchSandbox(); + sandbox.Files.Add("/work/.env"); + sandbox.FailMoves = true; + var runner = RunnerWithDefault(); + + var result = await runner.RunAsync(sandbox, "/work", "test", Cred()); + + Assert.False(result.Success); + Assert.Contains("refusing Unreal dispatch", result.Summary); + // Unreal was NEVER dispatched + Assert.DoesNotContain(sandbox.Execs, e => e.Argv.Count > 0 && e.Argv[0] == UnrealAgentRunner.DefaultBinary); + } + + [Fact] + public void ClassifyFailure_Exit0_IsNormal() + { + var runner = Runner(); + var result = new AgentResult(Success: true, Summary: "ok", Stdout: null, Stderr: null); + var classification = runner.ClassifyFailure(result); + + Assert.Equal(AgentFailureKind.Normal, classification.Kind); + } + + [Fact] + public void ClassifyFailure_Exit1_IsNormalWorkFailure() + { + var runner = Runner(); + var result = new AgentResult(Success: false, Summary: "agent exited 1", Stdout: null, Stderr: "some error"); + var classification = runner.ClassifyFailure(result); + + Assert.Equal(AgentFailureKind.Normal, classification.Kind); + } + + [Fact] + public void ClassifyFailure_Exit130_IsInfrastructureInterrupted() + { + var runner = Runner(); + var result = new AgentResult(Success: false, Summary: "agent exited 130", Stdout: null, Stderr: null); + var classification = runner.ClassifyFailure(result); + + Assert.Equal(AgentFailureKind.Infrastructure, classification.Kind); + Assert.Contains("130", classification.Reason); + } +} + +/// +/// Scripted for Unreal runner tests: +/// manages test -f/-e, mv, and unreal-agent-runner dispatch. +/// +internal sealed class UnrealDispatchSandbox : ISandbox +{ + public HashSet Files { get; } = new(StringComparer.Ordinal); + public int UnrealExitCode { get; set; } + public string UnrealStdout { get; set; } = "stdout"; + public string UnrealStderr { get; set; } = "stderr"; + public bool FailMoves { get; set; } + public string Id => "fake-unreal"; + public List Execs { get; } = []; + + public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + Execs.Add(exec); + var argv = exec.Argv; + + if (argv.Count == 3 && argv[0] == "test" && (argv[1] == "-f" || argv[1] == "-e")) + { + return Task.FromResult(Files.Contains(argv[2]) + ? new SandboxExecResult(0, string.Empty, string.Empty) + : new SandboxExecResult(1, string.Empty, string.Empty)); + } + + if (argv.Count == 4 && argv[0] == "mv" && argv[1] == "--") + { + if (FailMoves || !Files.Contains(argv[2])) + { + return Task.FromResult(new SandboxExecResult(1, string.Empty, $"mv: cannot stat '{argv[2]}'")); + } + + Files.Remove(argv[2]); + Files.Add(argv[3]); + return Task.FromResult(new SandboxExecResult(0, string.Empty, string.Empty)); + } + + if (argv.Count > 0 && argv[0] == UnrealAgentRunner.DefaultBinary) + { + exec.StdoutChunkCallback?.Invoke(UnrealStdout); + return Task.FromResult(new SandboxExecResult(UnrealExitCode, UnrealStdout, UnrealStderr)); + } + + return Task.FromResult(new SandboxExecResult(0, string.Empty, string.Empty)); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/tests/CodeyBox.Tests/UnrealCostExtractorTests.cs b/tests/CodeyBox.Tests/UnrealCostExtractorTests.cs new file mode 100644 index 000000000..b9e3dd897 --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealCostExtractorTests.cs @@ -0,0 +1,59 @@ +using CodeyBox.Agents.Unreal; +using CodeyBox.Core; + +namespace CodeyBox.Tests; + +public sealed class UnrealCostExtractorTests +{ + private readonly UnrealCostExtractor _extractor = new(); + + [Fact] + public void Kind_IsUnreal() + { + Assert.Equal(AgentKind.Unreal, _extractor.Kind); + } + + [Fact] + public void TryExtract_FromModelResponseUsage_ReturnsTokens() + { + const string stdout = + """ + {"Sequence":4,"Kind":"turn","Data":{}} + {"Sequence":5,"Kind":"model_response","Data":{"Response":{"Usage":{"InputTokens":10,"CachedInputTokens":3,"OutputTokens":5}}}} + """; + + var snapshot = _extractor.TryExtract(stdout, null); + + Assert.NotNull(snapshot); + Assert.Equal(10, snapshot.InputTokens); + Assert.Equal(3, snapshot.CachedInputTokens); + Assert.Equal(5, snapshot.OutputTokens); + } + + [Fact] + public void TryExtract_MultipleResponses_UsesHighestOrLatest() + { + const string stdout = + """ + {"Sequence":5,"Kind":"model_response","Data":{"Response":{"Usage":{"InputTokens":10,"CachedInputTokens":0,"OutputTokens":5}}}} + {"Sequence":9,"Kind":"model_response","Data":{"Response":{"Usage":{"InputTokens":25,"CachedInputTokens":5,"OutputTokens":12}}}} + """; + + var snapshot = _extractor.TryExtract(stdout, null); + + Assert.NotNull(snapshot); + Assert.Equal(25, snapshot.InputTokens); + Assert.Equal(5, snapshot.CachedInputTokens); + Assert.Equal(12, snapshot.OutputTokens); + } + + [Fact] + public void TryExtract_NoUsage_ReturnsNull() + { + const string stdout = "{\"Sequence\":1,\"Kind\":\"input\",\"Data\":{}}"; + + var snapshot = _extractor.TryExtract(stdout, null); + + Assert.Null(snapshot); + } +} diff --git a/tests/CodeyBox.Tests/UnrealProbeTests.cs b/tests/CodeyBox.Tests/UnrealProbeTests.cs new file mode 100644 index 000000000..ccbb744c5 --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealProbeTests.cs @@ -0,0 +1,153 @@ +using System.Text.Json; +using CodeyBox.Agents.Unreal; +using CodeyBox.Core; + +namespace CodeyBox.Tests; + +/// +/// Tests for Unreal probes: +/// - UnrealSmokeProbe (credential check + Codex rejection) +/// - UnrealInVmSmokeProbe (help step + stdin prompt dispatch step) +/// - UnrealModelListProbe (known model catalog) +/// +public sealed class UnrealProbeTests +{ + [Theory] + [InlineData("OPENROUTER_API_KEY")] + [InlineData("OPENAI_API_KEY")] + [InlineData("FIREWORKS_API_KEY")] + [InlineData("UNREAL_HARNESS_LLM_API_KEY")] + public async Task SmokeProbe_ValidApiKey_ReturnsOk(string keyName) + { + var probe = new UnrealSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary { [keyName] = "test-key-val" }, + new Dictionary()); + + var result = await probe.SmokeTestAsync(cred, CancellationToken.None); + + Assert.True(result.Ok); + Assert.Null(result.FailureReason); + Assert.Equal(SmokeFailureCategory.None, result.Category); + } + + [Fact] + public async Task SmokeProbe_MissingApiKey_ReturnsFail() + { + var probe = new UnrealSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary(), + new Dictionary()); + + var result = await probe.SmokeTestAsync(cred, CancellationToken.None); + + Assert.False(result.Ok); + Assert.Equal(UnrealAgentRunner.MissingCredentialMarker, result.FailureReason); + Assert.Equal(SmokeFailureCategory.Persistent, result.Category); + } + + [Fact] + public async Task SmokeProbe_CodexAccessToken_RejectsAsPersistent() + { + var probe = new UnrealSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary + { + ["OPENROUTER_API_KEY"] = "key", + ["OPENAI_CODEX_ACCESS_TOKEN"] = "token" + }, + new Dictionary()); + + var result = await probe.SmokeTestAsync(cred, CancellationToken.None); + + Assert.False(result.Ok); + Assert.Equal(UnrealAgentRunner.ProhibitedCodexCredentialMarker, result.FailureReason); + Assert.Equal(SmokeFailureCategory.Persistent, result.Category); + } + + [Fact] + public async Task SmokeProbe_CodexAuthFile_RejectsAsPersistent() + { + var probe = new UnrealSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary + { + ["OPENROUTER_API_KEY"] = "key", + ["OPENAI_CODEX_AUTH_FILE"] = "/path/to/auth.json" + }, + new Dictionary()); + + var result = await probe.SmokeTestAsync(cred, CancellationToken.None); + + Assert.False(result.Ok); + Assert.Equal(UnrealAgentRunner.ProhibitedCodexCredentialMarker, result.FailureReason); + Assert.Equal(SmokeFailureCategory.Persistent, result.Category); + } + + [Fact] + public async Task SmokeProbe_CodexProviderEnv_RejectsAsPersistent() + { + var probe = new UnrealSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary + { + ["OPENROUTER_API_KEY"] = "key", + ["UNREAL_HARNESS_LLM_PROVIDER"] = "openai-codex" + }, + new Dictionary()); + + var result = await probe.SmokeTestAsync(cred, CancellationToken.None); + + Assert.False(result.Ok); + Assert.Equal(UnrealAgentRunner.ProhibitedCodexCredentialMarker, result.FailureReason); + Assert.Equal(SmokeFailureCategory.Persistent, result.Category); + } + + [Fact] + public void InVmSmokeProbe_NullCredential_ReturnsSingleHelpStep() + { + var probe = new UnrealInVmSmokeProbe(); + var steps = probe.BuildSteps(credential: null); + + Assert.Single(steps); + Assert.Equal([UnrealAgentRunner.DefaultBinary, "-h"], steps[0].Argv); + Assert.Null(steps[0].Stdin); + } + + [Fact] + public void InVmSmokeProbe_WithCredential_ReturnsHelpStepAndStdinPromptStep() + { + var probe = new UnrealInVmSmokeProbe(); + var cred = new AgentCredential(AgentKind.Unreal, + new Dictionary { ["OPENROUTER_API_KEY"] = "key" }, + new Dictionary()); + + var steps = probe.BuildSteps(cred); + + Assert.Equal(2, steps.Count); + // Step 1: -h + Assert.Equal([UnrealAgentRunner.DefaultBinary, "-h"], steps[0].Argv); + + // Step 2: stdin prompt dispatch path + var step2 = steps[1]; + Assert.Equal(UnrealAgentRunner.DefaultBinary, step2.Argv[0]); + Assert.Contains("-workspace", step2.Argv); + Assert.Contains("-log-directory", step2.Argv); + Assert.Contains("-session-directory", step2.Argv); + Assert.NotNull(step2.Stdin); + + using var doc = JsonDocument.Parse(step2.Stdin); + Assert.Equal("ping", doc.RootElement.GetProperty("prompt").GetString()); + } + + [Fact] + public async Task ModelListProbe_ReturnsKnownModels() + { + var probe = new UnrealModelListProbe(); + var result = await probe.GetModelListAsync(CancellationToken.None); + + Assert.NotNull(result.ModelIds); + Assert.Contains("gpt-6-astra", result.ModelIds); + Assert.Contains("openrouter/nvidia/nemotron-3.5-lightning:free", result.ModelIds); + } +} diff --git a/tests/CodeyBox.Tests/UnrealQuotaFailureDetectorTests.cs b/tests/CodeyBox.Tests/UnrealQuotaFailureDetectorTests.cs new file mode 100644 index 000000000..0264ca4fa --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealQuotaFailureDetectorTests.cs @@ -0,0 +1,71 @@ +using CodeyBox.Agents; +using CodeyBox.Agents.Unreal; +using CodeyBox.Core; + +namespace CodeyBox.Tests; + +/// +/// Tests for : +/// - Detects rate-limit errors (429, rate limit exceeded) +/// - Detects quota limits (insufficient_quota, credit balance too low, key limit exceeded) +/// - Detects auth errors (unauthorized, invalid_api_key) +/// - Operator custom patterns support +/// +public sealed class UnrealQuotaFailureDetectorTests +{ + private readonly UnrealQuotaFailureDetector _detector = new(); + + [Fact] + public void Kind_IsUnreal() + { + Assert.Equal(AgentKind.Unreal, _detector.Kind); + } + + [Theory] + [InlineData("rate limit exceeded", QuotaFailureKind.RateLimitExceeded)] + [InlineData("error 429: too many requests", QuotaFailureKind.RateLimitExceeded)] + [InlineData("insufficient_quota for model", QuotaFailureKind.LimitReached)] + [InlineData("credit balance too low", QuotaFailureKind.LimitReached)] + [InlineData("unauthorized: invalid api key", QuotaFailureKind.Unauthorized)] + public void Detect_StandardPatterns_MatchesCorrectKind(string message, QuotaFailureKind expectedKind) + { + var detection = _detector.Detect(message, null); + Assert.NotNull(detection); + Assert.Equal(expectedKind, detection.Kind); + } + + [Fact] + public void Detect_JsonStdoutError_MatchesQuotaPattern() + { + const string stdout = + """ + {"Sequence":1,"Kind":"turn","Data":{}} + {"type":"error","message":"rate limit exceeded: please slow down requests"} + """; + + var detection = _detector.Detect(null, stdout); + Assert.NotNull(detection); + Assert.Equal(QuotaFailureKind.RateLimitExceeded, detection.Kind); + } + + [Fact] + public void Detect_HealthyOutput_ReturnsNull() + { + const string stdout = "{\"Sequence\":5,\"Kind\":\"model_response\",\"Data\":{}}"; + Assert.Null(_detector.Detect(null, stdout)); + Assert.Null(_detector.Detect(string.Empty, string.Empty)); + Assert.Null(_detector.Detect(null, null)); + } + + [Fact] + public void Detect_CustomOperatorPatterns_Matches() + { + var custom = new UnrealQuotaFailureDetector([ + new QuotaFailurePattern("custom-hard-limit-hit", QuotaFailureKind.LimitReached) + ]); + + var detection = custom.Detect("provider returned custom-hard-limit-hit", null); + Assert.NotNull(detection); + Assert.Equal(QuotaFailureKind.LimitReached, detection.Kind); + } +} diff --git a/tests/CodeyBox.Tests/UnrealStreamParserTests.cs b/tests/CodeyBox.Tests/UnrealStreamParserTests.cs new file mode 100644 index 000000000..3043f922e --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealStreamParserTests.cs @@ -0,0 +1,171 @@ +using System.Text.Json; +using CodeyBox.Agents; +using CodeyBox.Agents.Unreal; +using CodeyBox.Core; + +namespace CodeyBox.Tests; + +/// +/// Tests for : +/// - Real captured JSONL sample from pinned release v0.1.1 +/// - Multi-turn tool execution with tool_call and tool_call_status +/// - Token usage parsing from model_response +/// - Real error event {"type":"error","message":"..."} +/// - Claim discipline (claims Unreal session items, rejects other schemas) +/// +public sealed class UnrealStreamParserTests +{ + private readonly UnrealStreamParser _parser = new(); + + private const string SampleTurn1 = + """{"Sequence":1,"RecordedAt":"2026-09-24T17:09:01.949420242Z","Kind":"input","Data":{"ID":"afb61d14-8551-4b4e-ad8b-a838f29d0f0e","Kind":"control","Payload":{"Mode":"settings","Reason":"","Parameters":{"ReasoningEffort":"high"}}}}"""; + + private const string SampleTurn2 = + """{"Sequence":2,"RecordedAt":"2026-09-24T17:09:01.960298521Z","Kind":"input","Data":{"ID":"7fe3ec0c-df21-4e78-9375-a25ec96c82f4","Kind":"external","Payload":"test prompt"}}"""; + + private const string SampleTurn4 = + """{"Sequence":4,"RecordedAt":"2026-09-24T17:09:01.977842916Z","Kind":"turn","Data":{"ID":"5729508d-d7f6-491e-8240-87d224509d67","PreviousTurnID":"","Type":"regular"}}"""; + + private const string SampleModelResponseWithText = + """{"Sequence":5,"RecordedAt":"2026-09-24T17:09:01.987212341Z","Kind":"model_response","Data":{"TurnID":"5729508d-d7f6-491e-8240-87d224509d67","Response":{"ID":"resp-1","Stop":"complete","Output":[{"ProviderID":"","Type":"message","Data":{"Role":"assistant","Text":"Hello from unreal agent","Phase":""}}],"Usage":{"InputTokens":10,"CachedInputTokens":2,"CacheWriteInputTokens":0,"OutputTokens":5,"ReasoningTokens":0,"Raw":{"input_tokens":10,"output_tokens":5,"total_tokens":15}},"Failure":null}}}"""; + + private const string SampleModelResponseWithToolCall = + """{"Sequence":5,"RecordedAt":"2026-09-24T17:09:11.316959806Z","Kind":"model_response","Data":{"TurnID":"7687d080-a0f9-4aab-9a8d-d4b91c4e3f52","Response":{"ID":"resp-1","Stop":"complete","Output":[{"ProviderID":"","Type":"tool_call","Data":{"CallID":"call-1","Name":"Bash","Arguments":"{\"command\": \"echo hello > output.txt\"}"}}],"Usage":{"InputTokens":15,"CachedInputTokens":0,"CacheWriteInputTokens":0,"OutputTokens":8,"ReasoningTokens":0,"Raw":{"input_tokens":15,"output_tokens":8,"total_tokens":23}},"Failure":null}}}"""; + + private const string SampleToolCallStatusCompleted = + """{"Sequence":7,"RecordedAt":"2026-09-24T17:09:11.333677171Z","Kind":"tool_call_status","Data":{"TurnID":"7687d080-a0f9-4aab-9a8d-d4b91c4e3f52","CallID":"call-1","Status":{"Error":"","WaitingFor":["2dd39a7a-fd2e-4c95-a032-9646d1446361"]},"Operations":[{"MaxOutputLength":40000,"ID":"2dd39a7a-fd2e-4c95-a032-9646d1446361","Type":"shell","Version":3,"Status":"completed","State":{"Input":{"Command":"echo hello > output.txt","Shell":"/bin/sh","Directory":"/work"},"BaseDirectory":"/tmp","Phase":"","ProcessGroupID":0,"PendingExitCode":null,"OutSize":12,"ErrSize":0,"InlineOut":"","InlineErr":"","InlineOutTail":"","InlineErrTail":"","Result":{"Out":"hello\n","Err":"","OutSize":6,"ErrSize":0,"ExitCode":0},"TerminalError":"","ErrorTruncated":false,"OutTruncated":false,"ErrTruncated":false,"OutPath":"/tmp/out","ErrPath":"/tmp/err"}}]}}"""; + + private const string SampleRealErrorEvent = + """{"type":"error","message":"UNREAL_HARNESS_LLM_API_KEY or OPENAI_API_KEY must be set"}"""; + + [Fact] + public void Kind_IsUnreal() + { + Assert.Equal(AgentKind.Unreal, _parser.Kind); + } + + [Fact] + public void CanEmitShapeOf_MatchesUnrealOnly() + { + Assert.True(_parser.CanEmitShapeOf(AgentKind.Unreal)); + Assert.False(_parser.CanEmitShapeOf(AgentKind.Claude)); + Assert.False(_parser.CanEmitShapeOf(AgentKind.Codex)); + Assert.False(_parser.CanEmitShapeOf(AgentKind.Gemini)); + } + + [Fact] + public void TryClaim_UnrealSessionEvents_ClaimsTrue() + { + using var doc1 = JsonDocument.Parse(SampleTurn1); + Assert.True(_parser.TryClaim(doc1.RootElement)); + + using var doc5 = JsonDocument.Parse(SampleModelResponseWithText); + Assert.True(_parser.TryClaim(doc5.RootElement)); + + using var doc7 = JsonDocument.Parse(SampleToolCallStatusCompleted); + Assert.True(_parser.TryClaim(doc7.RootElement)); + } + + [Fact] + public void TryClaim_NonUnrealEvents_RejectsClaims() + { + // Claude / StreamJson frame + using var claudeDoc = JsonDocument.Parse("""{"type":"assistant","message":{"content":[]}}"""); + Assert.False(_parser.TryClaim(claudeDoc.RootElement)); + + // Codex frame + using var codexDoc = JsonDocument.Parse("""{"type":"turn","turn":{"turn_id":"1"}}"""); + Assert.False(_parser.TryClaim(codexDoc.RootElement)); + + // Bare error event (not a session item with Sequence/RecordedAt/Data) + using var errDoc = JsonDocument.Parse(SampleRealErrorEvent); + Assert.False(_parser.TryClaim(errDoc.RootElement)); + } + + private static Stream StreamOf(string text) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(text); + writer.Flush(); + stream.Position = 0; + return stream; + } + + [Fact] + public async Task ParseAsync_ModelResponse_ExtractsAssistantTextAndTokens() + { + var text = string.Join("\n", SampleTurn1, SampleTurn2, SampleTurn4, SampleModelResponseWithText); + var summary = await _parser.ParseAsync(StreamOf(text)); + + Assert.False(summary.IsUnsupported); + Assert.Equal("Hello from unreal agent", summary.FinalAssistantMessage); + Assert.Equal(10, summary.InputTokens); + Assert.Equal(2, summary.CachedInputTokens); + Assert.Equal(5, summary.OutputTokens); + } + + [Fact] + public async Task ParseAsync_ModelResponseWithToolCallAndStatus_ExtractsCompletedResult() + { + var text = string.Join("\n", SampleModelResponseWithToolCall, SampleToolCallStatusCompleted); + var summary = await _parser.ParseAsync(StreamOf(text)); + + Assert.False(summary.IsUnsupported); + Assert.Single(summary.ToolCalls); + var tool = summary.ToolCalls[0]; + Assert.Equal("call-1", tool.ToolUseId); + Assert.Equal("Bash", tool.ToolName); + Assert.Contains("echo hello", tool.InputSummary); + Assert.True(tool.Succeeded); + Assert.Equal(6, tool.OutputBytes); + Assert.Equal(15, summary.InputTokens); + Assert.Equal(8, summary.OutputTokens); + } + + [Fact] + public async Task ParseAsync_RealCapturedSession_MultiTurnExecution() + { + var line1 = SampleTurn1; + var line2 = SampleTurn2; + var line4 = SampleTurn4; + var line5 = SampleModelResponseWithToolCall; + var line7 = SampleToolCallStatusCompleted; + var line9 = """{"Sequence":9,"RecordedAt":"2026-09-24T17:09:11.33678647Z","Kind":"model_response","Data":{"TurnID":"deeae52f-2d0a-4891-9737-d377204222f2","Response":{"ID":"resp-2","Stop":"complete","Output":[{"ProviderID":"","Type":"message","Data":{"Role":"assistant","Text":"I have created output.txt","Phase":""}}],"Usage":{"InputTokens":20,"CachedInputTokens":0,"CacheWriteInputTokens":0,"OutputTokens":8,"ReasoningTokens":0,"Raw":{"input_tokens":20,"output_tokens":8,"total_tokens":28}},"Failure":null}}}"""; + + var text = string.Join("\n", line1, line2, line4, line5, line7, line9); + var summary = await _parser.ParseAsync(StreamOf(text)); + + Assert.False(summary.IsUnsupported); + Assert.Equal("I have created output.txt", summary.FinalAssistantMessage); + Assert.Single(summary.ToolCalls); + Assert.Equal("call-1", summary.ToolCalls[0].ToolUseId); + Assert.True(summary.ToolCalls[0].Succeeded); + Assert.Equal(20, summary.InputTokens); + Assert.Equal(8, summary.OutputTokens); + } + + [Fact] + public async Task ParseAsync_RealErrorEvent_ExtractsErrorMessage() + { + var summary = await _parser.ParseAsync(StreamOf(SampleRealErrorEvent)); + + Assert.False(summary.IsUnsupported); + Assert.NotNull(summary.FinalAssistantMessage); + Assert.Contains("UNREAL_HARNESS_LLM_API_KEY or OPENAI_API_KEY must be set", summary.FinalAssistantMessage); + } + + [Fact] + public async Task ParseAsync_EmptyStream_IsUnsupported() + { + var summary = await _parser.ParseAsync(StreamOf(string.Empty)); + Assert.True(summary.IsUnsupported); + } + + [Fact] + public async Task ParseAsync_NonJsonNoiseOnly_IsUnsupported() + { + var summary = await _parser.ParseAsync(StreamOf("random log line\nanother line\n")); + Assert.True(summary.IsUnsupported); + } +} diff --git a/tests/CodeyBox.Tests/UnrealTerminalDiagnoserTests.cs b/tests/CodeyBox.Tests/UnrealTerminalDiagnoserTests.cs new file mode 100644 index 000000000..7093e0fe6 --- /dev/null +++ b/tests/CodeyBox.Tests/UnrealTerminalDiagnoserTests.cs @@ -0,0 +1,53 @@ +using CodeyBox.Agents.Unreal; + +namespace CodeyBox.Tests; + +public sealed class UnrealTerminalDiagnoserTests +{ + [Fact] + public void TryExtractTerminalError_FromStdoutJson_ReturnsMessage() + { + const string stdout = + """ + {"Sequence":1,"RecordedAt":"2026-09-24T17:09:01Z","Kind":"input","Data":{}} + {"type":"error","message":"UNREAL_HARNESS_LLM_API_KEY or OPENAI_API_KEY must be set"} + """; + + var diag = UnrealTerminalDiagnoser.TryExtractTerminalError(stdout, stderr: null); + + Assert.Equal("UNREAL_HARNESS_LLM_API_KEY or OPENAI_API_KEY must be set", diag); + } + + [Fact] + public void TryExtractTerminalError_FromStderrLine_ReturnsMessage() + { + const string stderr = "unreal-agent-runner: provider rate limit exceeded (429)"; + + var diag = UnrealTerminalDiagnoser.TryExtractTerminalError(stdout: null, stderr); + + Assert.Equal("provider rate limit exceeded (429)", diag); + } + + [Fact] + public void TryExtractTerminalError_LongMessage_TruncatedTo500Chars() + { + var longMsg = new string('x', 600); + var stdout = $"{{\"type\":\"error\",\"message\":\"{longMsg}\"}}"; + + var diag = UnrealTerminalDiagnoser.TryExtractTerminalError(stdout, stderr: null); + + Assert.NotNull(diag); + Assert.Equal(500, diag.Length); + } + + [Fact] + public void TryExtractTerminalError_NoError_ReturnsNull() + { + const string stdout = "{\"Sequence\":1,\"Kind\":\"model_response\",\"Data\":{}}"; + const string stderr = "some informative log line"; + + var diag = UnrealTerminalDiagnoser.TryExtractTerminalError(stdout, stderr); + + Assert.Null(diag); + } +} From 4e8cd891266c6c28e9967b26b04f1da8c1c3cdcd Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Thu, 24 Sep 2026 19:35:09 +0000 Subject: [PATCH 2/3] codeybox: preempt checkpoint Add unreal-agent (Unreal Labs) as a supported agent runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeyBox-WorkItem: 04a7948457d64d448cee6cd2e854d8de CodeyBox-Agent: antigravity/gemini-3.8-flash-high CodeyBox-Fallbacks: antigravity→devin (×2 Agent antigravity rate-limited by provider (transient rate limit; retrying after backoff): agent exited 1) Co-Authored-By: CodeyBox From 3ca917eac82070a9c49a8d8d5a0255c3c2af8588 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Thu, 24 Sep 2026 20:02:48 +0000 Subject: [PATCH 3/3] Fix flaky audit-timeout attribution tests: elapse the idle budget on a fake clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AuditSandboxLaunchTimeoutEmitsStructuredEventAndAgentKind flaked under full-suite load: the wall-clock 100 ms AuditorIdleTimeout also governs audit sandbox launch and quiet setup execs (git clone), so a slow ProcessSandbox CreateAsync/clone tripped the idle guard for the build/test gate before the credentialed auditor's launch ever stalled — LastError then named test:build-and-test-pass instead of the intended auditor. Switch the four sub-second-idle-budget tests in this file to the established pipelineTimeProvider FakeTimeProvider + advancing-clock pattern already used throughout AuditPipelineIntegrationTests, so the budget cannot be consumed by unrelated real work under load. CodeyBox-Prompt-Revision: 1 Co-Authored-By: CodeyBox --- .../BuildTestGateAndAntiBiasTests.cs | 55 +++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/CodeyBox.Tests/BuildTestGateAndAntiBiasTests.cs b/tests/CodeyBox.Tests/BuildTestGateAndAntiBiasTests.cs index d99fd936b..7cac94bb1 100644 --- a/tests/CodeyBox.Tests/BuildTestGateAndAntiBiasTests.cs +++ b/tests/CodeyBox.Tests/BuildTestGateAndAntiBiasTests.cs @@ -6,6 +6,7 @@ using CodeyBox.Sandbox; using CodeyBox.Sandbox.Process; using Microsoft.Extensions.Logging.Abstractions; +using IdleClock = Microsoft.Extensions.Time.Testing.FakeTimeProvider; namespace CodeyBox.Tests; @@ -285,6 +286,8 @@ public async Task TimedOutBuildTestGateReturnsIncompleteVerdict_AndSkipsLlmPanel { AuditorIdleTimeout = TimeSpan.FromMilliseconds(100), }); + // Idle budget elapses on the fake clock (advanced below), not on wall-clock scheduling. + var idleClock = NewIdleClock(); using var tp = TestSupport.BuildPipeline( _workspace, @@ -293,12 +296,13 @@ public async Task TimedOutBuildTestGateReturnsIncompleteVerdict_AndSkipsLlmPanel maxAuditIterations: 1, credentials: AuditCredentials(), requiredBuildVerifier: TestRequiredBuildVerifier.NotApplicable, - pipelineTuning: tuning); + pipelineTuning: tuning, + pipelineTimeProvider: idleClock); tp.Agent.WorkPlan.Enqueue(new FileWrite("a.txt", "v1")); var item = NewItem(); await tp.Store.CreateAsync(item); - await tp.Pipeline.RunAsync(item, CancellationToken.None); + await RunWithAdvancingIdleClockAsync(idleClock, tp.Pipeline.RunAsync(item, CancellationToken.None)); var final = await tp.Store.GetAsync(item.Id); Assert.Equal(WorkItemState.Failed, final!.State); @@ -1159,6 +1163,8 @@ public async Task TimedOutBuildTestGateEmitsStructuredEventAndAgentKind() var auditAgentKind = new AgentKind("codex"); var extraRunners = new[] { new ScriptedAgent(Array.Empty()) { Kind = auditAgentKind } }; + // Idle budget elapses on the fake clock (advanced below), not on wall-clock scheduling. + var idleClock = NewIdleClock(); using var tp = TestSupport.BuildPipeline( _workspace, @@ -1176,12 +1182,13 @@ public async Task TimedOutBuildTestGateEmitsStructuredEventAndAgentKind() MaxIterations = 1 }, extraAgentRunners: extraRunners, + pipelineTimeProvider: idleClock, logger: captLogger); tp.Agent.WorkPlan.Enqueue(new FileWrite("a.txt", "v1")); var item = NewItem(); await tp.Store.CreateAsync(item); - await tp.Pipeline.RunAsync(item, CancellationToken.None); + await RunWithAdvancingIdleClockAsync(idleClock, tp.Pipeline.RunAsync(item, CancellationToken.None)); var final = await tp.Store.GetAsync(item.Id); Assert.Equal(WorkItemState.Failed, final!.State); @@ -1219,6 +1226,10 @@ public async Task AuditSandboxLaunchTimeoutEmitsStructuredEventAndAgentKind() var extraRunners = new[] { new ScriptedAgent(Array.Empty()) { Kind = auditAgentKind } }; var defaultProvider = new ProcessSandboxProvider(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); var launchTimeoutProvider = new AuditCredentialLaunchTimeoutSandboxProvider(defaultProvider); + // Idle budget elapses on the fake clock (advanced below), not on wall-clock + // scheduling — a slow non-credentialed sandbox launch/setup under suite load + // must not trip the 100 ms budget before the credentialed launch does. + var idleClock = NewIdleClock(); using var tp = TestSupport.BuildPipeline( _workspace, @@ -1237,12 +1248,13 @@ public async Task AuditSandboxLaunchTimeoutEmitsStructuredEventAndAgentKind() }, extraAgentRunners: extraRunners, sandboxProvider: launchTimeoutProvider, + pipelineTimeProvider: idleClock, logger: captLogger); tp.Agent.WorkPlan.Enqueue(new FileWrite("a.txt", "v1")); var item = NewItem(); await tp.Store.CreateAsync(item); - await tp.Pipeline.RunAsync(item, CancellationToken.None); + await RunWithAdvancingIdleClockAsync(idleClock, tp.Pipeline.RunAsync(item, CancellationToken.None)); var final = await tp.Store.GetAsync(item.Id); Assert.Equal(WorkItemState.Failed, final!.State); @@ -1280,6 +1292,8 @@ public async Task TimedOutAuditorLogAttributionOnTeardownFailures() // Wrap the default sandbox provider to force kill/dispose timeouts var defaultProvider = new ProcessSandboxProvider(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); await using var timeoutProvider = new TimeoutSandboxProvider(defaultProvider, forceKillTimeout: true, forceDisposeTimeout: true); + // Idle budget elapses on the fake clock (advanced below), not on wall-clock scheduling. + var idleClock = NewIdleClock(); using var tp = TestSupport.BuildPipeline( _workspace, @@ -1298,12 +1312,13 @@ public async Task TimedOutAuditorLogAttributionOnTeardownFailures() }, extraAgentRunners: extraRunners, sandboxProvider: timeoutProvider, + pipelineTimeProvider: idleClock, logger: captLogger); tp.Agent.WorkPlan.Enqueue(new FileWrite("a.txt", "v1")); var item = NewItem(); await tp.Store.CreateAsync(item); - await tp.Pipeline.RunAsync(item, CancellationToken.None); + await RunWithAdvancingIdleClockAsync(idleClock, tp.Pipeline.RunAsync(item, CancellationToken.None)); var final = await tp.Store.GetAsync(item.Id); Assert.Equal(WorkItemState.Failed, final!.State); @@ -1353,6 +1368,36 @@ private static void MakeExecutable(string path) UnixFileMode.OtherRead | UnixFileMode.OtherExecute); } + // Sub-second AuditorIdleTimeout budgets in this class elapse on a fake + // clock (pipelineTimeProvider), advanced in lockstep with real time by + // RunWithAdvancingIdleClockAsync while the pipeline runs. A wall-clock + // 100 ms budget is a race under parallel-suite load: a genuinely slow + // sandbox launch or quiet audit-setup exec (e.g. git clone) trips the + // idle guard and the timeout is attributed to whichever auditor was being + // provisioned — not the auditor the test intends to time out. With the + // fake clock the budget only advances while the advancer runs, and under + // load the advancer's real delays stretch while its fake step stays fixed, + // so timeouts fire late rather than spuriously early. The clock starts at + // the real now so fake timestamps stay comparable with the real-clock + // timestamps recorded by stores outside the pipeline clock. + private static IdleClock NewIdleClock() => new(DateTimeOffset.UtcNow); + + private static async Task RunWithAdvancingIdleClockAsync(IdleClock clock, Task pipelineTask) + { + ArgumentNullException.ThrowIfNull(clock); + ArgumentNullException.ThrowIfNull(pipelineTask); + // 10 ms fake per ~10 ms real — never faster. A faster rate would + // tighten every pipeline budget in real terms and kill real work + // (e.g. audit-setup git clone) spuriously. + while (!pipelineTask.IsCompleted) + { + clock.Advance(TimeSpan.FromMilliseconds(10)); + await Task.Delay(10); + } + + await pipelineTask; + } + private static ConstantCredentialProvider AuditCredentials() => new(new AgentCredential( AgentKind.Claude,