From dcf47a90c77f5ae3ccd173b8d93bfa38aa27a0f7 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Fri, 18 Sep 2026 06:26:19 +0000 Subject: [PATCH] Materialise incoming agent credentials on mid-iteration runner swaps Work item a09d2275: a phase entered under agent A and switched to agent B mid-iteration executed B without B's credentials. Direct credential variables are baked into the sandbox spec at creation, the reusable sandbox kept serving A's baked environment after the swap, and the resulting provider 401 was misreported as 'agent requires re-authentication'. - New AgentRunnerSwitchGate: the single seam for every post-creation runner switch. AssessSwitch re-validates the incoming runner against its own credential (agent match + runner's own environment classification); ScopeSandbox + CollectCredentialEnvironmentScope + ValidateCredentialScope carry the conflict-resolver credential-scoping mechanism unchanged; ToPostSwapInfrastructureFailure reclassifies a first-attempt post-swap 401 as infrastructure with the 401 evidence preserved. - Quota-fallback path: each fallback candidate is bound, credential-resolved and gate-assessed before selection; unmaterialisable candidates are refused (logged, added to tried keys, no dispatch) and an all-refused field keeps the original failure. Every successful swap arms a one-attempt guard that converts a post-swap AgentAuthRequiredException to infrastructure, and surrenders the warm reusable sandbox so the retry provisions a fresh sandbox from the incoming member's spec. Genuine expiry with no preceding swap still fails as auth-required. - Conflict-resolver path: candidates are partitioned through the same gate; refused candidates are named on the attempt trail and never dispatch, and the sandbox scoping now flows through the shared helper. Null-credential candidates remain dispatchable on both paths (ambient auth + runtime auth detection own that outcome). - Tests: 11 new AgentRunnerSwitchTests cover gate allow/refuse/secret-safety, end-to-end swap credential materialisation (per-attempt sandbox specs), refusal with original failure preserved and no dispatch, post-swap 401 as infrastructure, no-swap auth still auth-required, and both resolver-path behaviours. Verification: dotnet build --no-incremental /warnaserror clean; full dotnet test suite green (13360 passed in CodeyBox.Tests); gitleaks clean. semgrep is not installed in this environment (no pip), so that scan could not be run. CodeyBox-Prompt-Revision: 1 Co-Authored-By: CodeyBox --- .../AgentRunnerSwitchGate.cs | 333 ++++++++ .../AgenticConflictResolver.cs | 189 +---- .../PipelineRunner.QuotaFallback.cs | 124 ++- .../CodeyBox.Tests/AgentRunnerSwitchTests.cs | 773 ++++++++++++++++++ 4 files changed, 1259 insertions(+), 160 deletions(-) create mode 100644 src/CodeyBox.Orchestrator/AgentRunnerSwitchGate.cs create mode 100644 tests/CodeyBox.Tests/AgentRunnerSwitchTests.cs diff --git a/src/CodeyBox.Orchestrator/AgentRunnerSwitchGate.cs b/src/CodeyBox.Orchestrator/AgentRunnerSwitchGate.cs new file mode 100644 index 000000000..527319e6a --- /dev/null +++ b/src/CodeyBox.Orchestrator/AgentRunnerSwitchGate.cs @@ -0,0 +1,333 @@ +using CodeyBox.Core; +using CodeyBox.Sandbox; + +namespace CodeyBox.Orchestrator; + +/// +/// The single seam every path that replaces the executing agent runner after +/// sandbox creation must pass through. Two defect shapes motivate it: +/// +/// +/// Direct credential environment variables are baked into a sandbox +/// spec at creation time for exactly one agent kind. A runner swapped in +/// afterwards that never had its own credential materialised into the +/// environment it actually executes with runs unauthenticated and 401s — +/// which the auth-failure detector then misreports as "agent requires +/// re-authentication" even though the credential itself is healthy. +/// +/// A swap that cannot work is not a fallback: dispatching an agent +/// whose credential belongs to another agent or fails the runner's own +/// environment classification is a guaranteed failure. The swap must be +/// refused and the original failure kept instead. +/// +/// +/// +/// Both the mid-iteration quota-fallback path +/// (PipelineRunner.InvokeAgentWithQuotaFallbackAsync) and the shared- +/// sandbox conflict-resolver path () +/// assess every incoming runner through and scope +/// shared sandboxes through , so the next path +/// that swaps runners inherits the same behaviour instead of reintroducing +/// the bug a third time. The fallback path additionally surrenders the warm +/// reusable sandbox on every swap: direct credential variables are baked +/// into the sandbox spec at creation, so a reused sandbox would still carry +/// the exhausted member's environment no matter what the new spec says. +/// +/// +/// Credential exposure is unchanged by this gate: the incoming agent is +/// assessed against — and later executed with — its own credential only. The +/// existing agent-match rule stays; the gate re-evaluates the match against +/// the agent that will actually run. Refusal reasons carry variable names +/// and runner identities only, never secret values. +/// +internal static class AgentRunnerSwitchGate +{ + /// + /// Upper bound on scoped credential environment names. Matches the + /// sandbox exec cap on unset variables so a scope can always be applied. + /// + public const int MaximumScopedCredentialEnvironmentVariables = + SandboxExec.MaximumEnvironmentVariablesToUnset; + + /// + /// Outcome of assessing one incoming runner. is + /// true only when the runner may be dispatched with + /// — a null credential is always allowed (there + /// is nothing to materialise; runtime auth detection owns the outcome). + /// A refused assessment carries a short operator-safe + /// naming the runner and the materialisation + /// defect — never a secret value. + /// + public sealed record SwitchAssessment( + bool Allowed, + AgentCredential? Credential, + string? RefusalReason); + + /// + /// Decides whether may be dispatched + /// with . Pure except for the + /// runner's own credential-environment classification (the same + /// SandboxEnvironmentVariablePolicy check sandbox-spec building + /// applies), so it never admits a swap the execution environment would + /// reject — or silently run unauthenticated — later. + /// + public static SwitchAssessment AssessSwitch( + IAgentRunner incomingRunner, + AgentCredential? incomingCredential) + { + ArgumentNullException.ThrowIfNull(incomingRunner); + var kind = incomingRunner.Kind; + + if (incomingCredential is null) + { + // No credential to materialise. This is not a refusal: runners + // without credential environment (test doubles, local-only CLIs) + // need nothing, and runners WITH declared environment may still + // authenticate from image-baked or ambient sandbox state — the + // resolver and fallback paths both historically dispatch such + // candidates and let runtime auth detection own the outcome. A + // 401 that follows a swap is reclassified by + // ToPostSwapInfrastructureFailure instead of refused here, so a + // missing credential can never surface as "requires + // re-authentication" after a runner change. + return new SwitchAssessment(Allowed: true, Credential: null, RefusalReason: null); + } + + if (incomingCredential.Agent != kind) + { + return new SwitchAssessment( + Allowed: false, + Credential: incomingCredential, + RefusalReason: + $"credential belongs to agent '{incomingCredential.Agent.Value}', " + + $"not '{kind.Value}'"); + } + + try + { + _ = SandboxEnvironmentVariablePolicy.SelectDirectCredentialEnvironment( + incomingCredential, + incomingRunner, + nameof(AgentCredential.EnvironmentVariables)); + } + catch (ArgumentException ex) + { + return new SwitchAssessment( + Allowed: false, + Credential: incomingCredential, + RefusalReason: + $"credential for agent '{kind.Value}' cannot be materialised: {ex.Message}"); + } + + return new SwitchAssessment(Allowed: true, Credential: incomingCredential, RefusalReason: null); + } + + /// + /// Throws when + /// may not be used with + /// . Same check as + /// in throwing form, for call sites (credential-file staging) that + /// historically fail the candidate with an exception rather than a + /// refusal record. + /// + public static void ValidateCredentialScope(IAgentRunner runner, AgentCredential? credential) + { + ArgumentNullException.ThrowIfNull(runner); + if (credential is { } cred && cred.Agent != runner.Kind) + { + throw new AgentCredentialScopeException( + runner.Kind, + $"credential belongs to agent '{cred.Agent.Value}'"); + } + } + + /// + /// Collects the union of credential environment names across already- + /// assessed candidates so a shared sandbox can strip every credential + /// name before re-adding only the current candidate's direct values + /// (see ). Candidates are pre-validated pairs; + /// entries with no environment variables contribute nothing. Throws only + /// for a null entry (programming error) or an aggregate over the sandbox + /// exec unset cap (which would make the scope unappliable). + /// + public static IReadOnlySet CollectCredentialEnvironmentScope( + IEnumerable<(IAgentRunner Runner, AgentCredential? Credential)> assessed, + string paramName) + { + ArgumentNullException.ThrowIfNull(assessed); + var names = new HashSet(StringComparer.Ordinal); + foreach (var (runner, credential) in assessed) + { + if (runner is null) + throw new ArgumentException("Agent runner entries cannot contain null entries.", paramName); + if (credential is not { EnvironmentVariables.Count: > 0 }) + continue; + _ = SandboxEnvironmentVariablePolicy.SelectDirectCredentialEnvironment( + credential, + runner, + nameof(AgentCredential.EnvironmentVariables)); + + foreach (var name in credential.EnvironmentVariables.Keys) + { + names.Add(name); + if (names.Count > MaximumScopedCredentialEnvironmentVariables) + { + throw new ArgumentException( + $"Agent candidates cannot declare more than {MaximumScopedCredentialEnvironmentVariables} credential environment variables in aggregate.", + paramName); + } + } + } + return names; + } + + /// + /// Scopes a shared to one assessed candidate: + /// every scoped credential name is removed from each launched process and + /// only the candidate's declared direct values are re-added. File-backed + /// values stay confined to the runner's stdin materialisation path even + /// when an older caller provisioned them in the sandbox's base + /// environment. Returns the sandbox untouched when the scope is empty. + /// + public static ISandbox ScopeSandbox( + ISandbox sandbox, + IAgentRunner runner, + AgentCredential? credential, + IReadOnlySet credentialEnvironmentNames) + { + ArgumentNullException.ThrowIfNull(sandbox); + ArgumentNullException.ThrowIfNull(runner); + ArgumentNullException.ThrowIfNull(credentialEnvironmentNames); + if (credentialEnvironmentNames.Count == 0) + return sandbox; + + var directEnvironment = credential is { } cred + ? SandboxEnvironmentVariablePolicy.SelectDirectCredentialEnvironment( + cred, + runner, + nameof(AgentCredential.EnvironmentVariables)) + : new Dictionary(StringComparer.Ordinal); + + return new AgentCredentialScopedSandbox( + sandbox, + credentialEnvironmentNames, + directEnvironment); + } + + /// + /// Reclassifies an authentication failure observed on the first attempt + /// immediately after a runner swap as infrastructure. The swapped-in + /// agent was assessed as materialisable before dispatch, so a 401 on its + /// very first attempt far more likely means the swap executed without + /// the agent's credential material than a simultaneously-expired + /// credential — and reporting it as "agent requires re-authentication" + /// sends the operator to check a healthy credential while failing the + /// item as though the agent needed re-auth. The original message + /// (already redacted upstream) is preserved verbatim after the swap + /// context prefix, and the original exception is kept as + /// so the 401 evidence is not + /// lost. Auth failures with no preceding swap are untouched: a genuinely + /// expired credential still fails the item as + /// . + /// + public static AgentInfrastructureFailureException ToPostSwapInfrastructureFailure( + AgentAuthRequiredException authFailure, + string phase) + { + ArgumentNullException.ThrowIfNull(authFailure); + return new AgentInfrastructureFailureException( + authFailure.Agent, + phase, + $"Agent '{authFailure.Agent.Value}' reported provider authentication failure " + + $"immediately after a mid-iteration runner swap in phase '{phase}'; " + + "treating as infrastructure (the swapped-in agent likely executed without " + + "its own credentials) rather than an agent re-authentication requirement: " + + authFailure.Message, + authFailure); + } +} + +/// +/// Scopes a shared sandbox to one agent-switch candidate. Every non-current +/// credential name is removed from each launched process; only the current +/// candidate's declared direct values survive. File-backed values therefore +/// remain confined to the stdin materialisation path even when an older +/// caller accidentally provisioned them in the sandbox's base environment. +/// +internal sealed class AgentCredentialScopedSandbox : ISandboxDecorator +{ + private readonly ISandbox _inner; + private readonly IReadOnlySet _credentialEnvironmentNames; + private readonly IReadOnlyDictionary _directEnvironment; + + public AgentCredentialScopedSandbox( + ISandbox inner, + IReadOnlySet credentialEnvironmentNames, + IReadOnlyDictionary directEnvironment) + { + _inner = inner; + _credentialEnvironmentNames = credentialEnvironmentNames; + _directEnvironment = directEnvironment; + } + + public ISandbox InnerSandbox => _inner; + public string Id => _inner.Id; + public SandboxAgentOutputTransportKind AgentOutputTransportKind => _inner.AgentOutputTransportKind; + public SandboxBatchLaunchMode BatchLaunchMode => _inner.BatchLaunchMode; + public SandboxResourceMetrics? ResourceMetrics => _inner.ResourceMetrics; + + public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(exec); + var environment = exec.ExtraEnvironment is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(exec.ExtraEnvironment, StringComparer.Ordinal); + foreach (var (name, value) in _directEnvironment) + environment[name] = value; + + var removals = exec.EnvironmentVariablesToUnset.ToHashSet(StringComparer.Ordinal); + foreach (var name in _credentialEnvironmentNames) + { + if (!_directEnvironment.ContainsKey(name)) + removals.Add(name); + } + if (removals.Count > SandboxExec.MaximumEnvironmentVariablesToUnset) + { + throw new ArgumentException( + $"Candidate credential scope cannot unset more than {SandboxExec.MaximumEnvironmentVariablesToUnset} environment variables.", + nameof(exec)); + } + + return _inner.ExecAsync(exec with + { + ExtraEnvironment = environment.Count == 0 ? null : environment, + EnvironmentVariablesToUnset = removals.Order(StringComparer.Ordinal).ToArray(), + EnvironmentContainsSecrets = exec.EnvironmentContainsSecrets || _directEnvironment.Count > 0, + }, ct); + } + + public Task SyncStateToHostAsync(CancellationToken ct = default) => + _inner.SyncStateToHostAsync(ct); + + public Task KillActiveExecsAsync(CancellationToken ct = default) => + _inner.KillActiveExecsAsync(ct); + + public Task GetScreenshotAsync(CancellationToken ct = default) => + _inner.GetScreenshotAsync(ct); + + public Task SynthesizeInputAsync( + IReadOnlyList events, + CancellationToken ct = default) => + _inner.SynthesizeInputAsync(events, ct); + + public Task GetAccessibilityAtPointAsync( + int x, + int y, + CancellationToken ct = default) => + _inner.GetAccessibilityAtPointAsync(x, y, ct); + + public Task GetAccessibilityTreeJsonAsync(CancellationToken ct = default) => + _inner.GetAccessibilityTreeJsonAsync(ct); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/CodeyBox.Orchestrator/AgenticConflictResolver.cs b/src/CodeyBox.Orchestrator/AgenticConflictResolver.cs index 1c0095c97..0419fd112 100644 --- a/src/CodeyBox.Orchestrator/AgenticConflictResolver.cs +++ b/src/CodeyBox.Orchestrator/AgenticConflictResolver.cs @@ -208,8 +208,6 @@ public sealed record AgenticConflictCandidatesResult( /// public sealed class AgenticConflictResolver { - private const int MaximumScopedCredentialEnvironmentVariables = - SandboxExec.MaximumEnvironmentVariablesToUnset; private readonly AgenticConflictResolverOptionsSnapshot _options; private readonly ILogger _log; private readonly Func? _credentialFileMaterialiser; @@ -289,7 +287,30 @@ public async Task ResolveAsync( foreach (var file in conflictFiles) MergeConflictPathInspector.ValidateRelativeWorkPath(file); - var credentialEnvironmentNames = BuildCredentialEnvironmentScope(candidates); + // Every runner switch — including each resolver candidate — goes + // through the shared agent-switch gate, the same seam the + // mid-iteration quota-fallback path uses. A candidate whose + // credential cannot be materialised (wrong agent, unclassifiable + // variables) is refused up front: it never dispatches and never + // touches the shared sandbox, while viable candidates still run. + var viableCandidates = new List(candidates.Count); + var refusedCandidates = new List<(AgenticConflictResolverCandidate Candidate, string Reason)>(); + foreach (var candidate in candidates) + { + if (candidate is null) + throw new ArgumentException("Agent candidates cannot contain null entries.", nameof(candidates)); + var assessment = AgentRunnerSwitchGate.AssessSwitch(candidate.Runner, candidate.Credential); + if (!assessment.Allowed) + { + refusedCandidates.Add((candidate, assessment.RefusalReason ?? "credential cannot be materialised")); + continue; + } + viableCandidates.Add(candidate); + } + + var credentialEnvironmentNames = AgentRunnerSwitchGate.CollectCredentialEnvironmentScope( + viableCandidates.Select(static c => (c.Runner, c.Credential)), + nameof(candidates)); // Gate the start-of-resolve log on the pipeline-supplied hint. The // resolver is generic conflict machinery; it should not know which @@ -304,10 +325,17 @@ public async Task ResolveAsync( var options = _options.Current; var maxIterations = Math.Max(1, options.MaxIterations); var maxAttemptsPerAgent = Math.Max(1, options.MaxAttemptsPerAgent); - var maxQuality = candidates.Max(c => c.QualityScore); + var maxQuality = viableCandidates.Count == 0 ? 0 : viableCandidates.Max(c => c.QualityScore); var triedStrongest = false; var attemptTrail = new List(); + foreach (var (refused, reason) in refusedCandidates) + { + _log.LogWarning( + "Agentic conflict resolver: refusing candidate '{Agent}' for {WorkItemId} ({Reason}); skipping without dispatch", + refused.Runner.Kind.Value, workItemId, reason); + attemptTrail.Add($"{refused.Runner.Kind.Value}#0(credential refused: {Truncate(reason, 120)})"); + } var authFailures = new List(); int totalIterations = 0; AgentResult? lastAgentResult = null; @@ -422,7 +450,7 @@ void RecordAuthRequiredAttemptFailure( totalIterations = Math.Max(0, totalIterations - 1); } - foreach (var candidate in candidates) + foreach (var candidate in viableCandidates) { if (totalIterations >= maxIterations) { @@ -431,9 +459,10 @@ void RecordAuthRequiredAttemptFailure( var runner = candidate.Runner; var isStrongest = candidate.QualityScore == maxQuality; - var candidateSandbox = CreateCandidateCredentialSandbox( + var candidateSandbox = AgentRunnerSwitchGate.ScopeSandbox( sandbox, - candidate, + candidate.Runner, + candidate.Credential, credentialEnvironmentNames); if (previousScopedCandidate is { } previousCandidate) @@ -480,7 +509,7 @@ await ClearCandidateCredentialFilesAsync( { try { - ValidateCandidateCredentialScope(candidate); + AgentRunnerSwitchGate.ValidateCredentialScope(candidate.Runner, candidate.Credential); await _credentialFileMaterialiser(sandbox, candidate.Credential, ct).ConfigureAwait(false); } catch (OperationCanceledException) @@ -810,151 +839,7 @@ void RecordCredentialSetupFailure(string stage, Exception ex) }; } - private static void ValidateCandidateCredentialScope(AgenticConflictResolverCandidate candidate) - { - if (candidate.Credential is { } credential && credential.Agent != candidate.Runner.Kind) - { - throw new AgentCredentialScopeException( - candidate.Runner.Kind, - $"credential belongs to agent '{credential.Agent.Value}'"); - } - } - - private static IReadOnlySet BuildCredentialEnvironmentScope( - IReadOnlyList candidates) - { - var names = new HashSet(StringComparer.Ordinal); - foreach (var candidate in candidates) - { - if (candidate is null) - throw new ArgumentException("Agent candidates cannot contain null entries.", nameof(candidates)); - ValidateCandidateCredentialScope(candidate); - if (candidate.Credential is not { EnvironmentVariables.Count: > 0 } credential) - continue; - _ = SandboxEnvironmentVariablePolicy.SelectDirectCredentialEnvironment( - credential, - candidate.Runner, - nameof(AgentCredential.EnvironmentVariables)); - - foreach (var name in credential.EnvironmentVariables.Keys) - { - names.Add(name); - if (names.Count > MaximumScopedCredentialEnvironmentVariables) - { - throw new ArgumentException( - $"Resolver candidates cannot declare more than {MaximumScopedCredentialEnvironmentVariables} credential environment variables in aggregate.", - nameof(candidates)); - } - } - } - return names; - } - - private static ISandbox CreateCandidateCredentialSandbox( - ISandbox sandbox, - AgenticConflictResolverCandidate candidate, - IReadOnlySet credentialEnvironmentNames) - { - if (credentialEnvironmentNames.Count == 0) - return sandbox; - - var directEnvironment = candidate.Credential is { } credential - ? SandboxEnvironmentVariablePolicy.SelectDirectCredentialEnvironment( - credential, - candidate.Runner, - nameof(AgentCredential.EnvironmentVariables)) - : new Dictionary(StringComparer.Ordinal); - - return new CandidateCredentialSandbox( - sandbox, - credentialEnvironmentNames, - directEnvironment); - } - - /// - /// Scopes a shared resolver sandbox to one candidate. Every non-current - /// credential name is removed from each launched process; only the current - /// candidate's declared direct values survive. File-backed values therefore - /// remain confined to the stdin materialisation path even when an older - /// caller accidentally provisioned them in the sandbox's base environment. - /// - private sealed class CandidateCredentialSandbox : ISandboxDecorator - { - private readonly ISandbox _inner; - private readonly IReadOnlySet _credentialEnvironmentNames; - private readonly IReadOnlyDictionary _directEnvironment; - - public CandidateCredentialSandbox( - ISandbox inner, - IReadOnlySet credentialEnvironmentNames, - IReadOnlyDictionary directEnvironment) - { - _inner = inner; - _credentialEnvironmentNames = credentialEnvironmentNames; - _directEnvironment = directEnvironment; - } - - public ISandbox InnerSandbox => _inner; - public string Id => _inner.Id; - public SandboxAgentOutputTransportKind AgentOutputTransportKind => _inner.AgentOutputTransportKind; - public SandboxBatchLaunchMode BatchLaunchMode => _inner.BatchLaunchMode; - public SandboxResourceMetrics? ResourceMetrics => _inner.ResourceMetrics; - - public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) - { - ArgumentNullException.ThrowIfNull(exec); - var environment = exec.ExtraEnvironment is null - ? new Dictionary(StringComparer.Ordinal) - : new Dictionary(exec.ExtraEnvironment, StringComparer.Ordinal); - foreach (var (name, value) in _directEnvironment) - environment[name] = value; - - var removals = exec.EnvironmentVariablesToUnset.ToHashSet(StringComparer.Ordinal); - foreach (var name in _credentialEnvironmentNames) - { - if (!_directEnvironment.ContainsKey(name)) - removals.Add(name); - } - if (removals.Count > SandboxExec.MaximumEnvironmentVariablesToUnset) - { - throw new ArgumentException( - $"Candidate credential scope cannot unset more than {SandboxExec.MaximumEnvironmentVariablesToUnset} environment variables.", - nameof(exec)); - } - - return _inner.ExecAsync(exec with - { - ExtraEnvironment = environment.Count == 0 ? null : environment, - EnvironmentVariablesToUnset = removals.Order(StringComparer.Ordinal).ToArray(), - EnvironmentContainsSecrets = exec.EnvironmentContainsSecrets || _directEnvironment.Count > 0, - }, ct); - } - - public Task SyncStateToHostAsync(CancellationToken ct = default) => - _inner.SyncStateToHostAsync(ct); - - public Task KillActiveExecsAsync(CancellationToken ct = default) => - _inner.KillActiveExecsAsync(ct); - - public Task GetScreenshotAsync(CancellationToken ct = default) => - _inner.GetScreenshotAsync(ct); - public Task SynthesizeInputAsync( - IReadOnlyList events, - CancellationToken ct = default) => - _inner.SynthesizeInputAsync(events, ct); - - public Task GetAccessibilityAtPointAsync( - int x, - int y, - CancellationToken ct = default) => - _inner.GetAccessibilityAtPointAsync(x, y, ct); - - public Task GetAccessibilityTreeJsonAsync(CancellationToken ct = default) => - _inner.GetAccessibilityTreeJsonAsync(ct); - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; - } private static async Task ClearCandidateCredentialFilesAsync( ISandbox sandbox, diff --git a/src/CodeyBox.Orchestrator/PipelineRunner.QuotaFallback.cs b/src/CodeyBox.Orchestrator/PipelineRunner.QuotaFallback.cs index 7be236b29..63af0ab04 100644 --- a/src/CodeyBox.Orchestrator/PipelineRunner.QuotaFallback.cs +++ b/src/CodeyBox.Orchestrator/PipelineRunner.QuotaFallback.cs @@ -477,6 +477,14 @@ await _quotaClassifier.RecordIfQuotaFailureAsync( var triedCount = 0; DateTimeOffset? earliestReset = null; var sawQuotaBlockedCandidate = false; + // Armed by every successful runner swap and consumed by the next + // attempt's outcome: an authentication failure on the first attempt + // immediately after a swap is classified as infrastructure (the + // swapped-in agent likely executed without its own credentials), + // never as an agent re-authentication requirement. Disarmed by any + // completed post-swap attempt or by the conversion itself, so genuine + // credential expiry on later attempts still fails as auth-required. + var postSwapAuthGuardArmed = false; var currentRunner = initialRunner; var currentItem = initialItem; AgentKind? pausedFallbackAgent = null; @@ -551,6 +559,7 @@ async Task MoveToNextMemberOrThrowAsync( // Find the next candidate that we haven't already tried this run. var candidates = await _classRouter.OrderedFallbackCandidatesAsync(item, project, ct, fallbackSmokeTarget); AgentMembership? nextMember = null; + IAgentRunner? nextRunner = null; foreach (var candidate in candidates) { var key = TriedMemberKey(candidate); @@ -568,7 +577,7 @@ async Task MoveToNextMemberOrThrowAsync( classId, candidate.Agent.Value, requireCapability, item.Id); continue; } - if (!_agents.TryGet(candidate.Agent, out _)) + if (!_agents.TryGet(candidate.Agent, out var candidateRunner)) { // Audible misconfiguration: class declares this agent kind but // no runner is wired in DI; skipping silently would hide the gap. @@ -577,6 +586,58 @@ async Task MoveToNextMemberOrThrowAsync( classId, candidate.Agent.Value, item.Id); continue; } + // Agent-switch credential gate (the same seam the + // conflict-resolver path uses): the incoming runner must be + // dispatchable with its OWN credentials before it is + // selected. A swap that cannot work is not a fallback — + // dispatching it would run the agent unauthenticated and + // 401, so the candidate is refused and the search continues. + // When every candidate is refused, the no-candidate branch + // below keeps the original failure with no dispatch attempted. + var candidateTrialItem = item with + { + Agent = candidate.Agent, + AgentInstanceId = candidate.RouteKey, + ModelId = candidate.ModelId, + ReasoningMode = candidate.ReasoningMode, + }; + IAgentRunner boundCandidateRunner; + try + { + boundCandidateRunner = BindMemberRunner(candidateRunner, candidate); + } + catch (Exception bindEx) when (bindEx is not OperationCanceledException) + { + _log.LogWarning(bindEx, + "Class '{ClassId}' member {Agent}/{Model} cannot bind its runner configuration; refusing for fallback (work item {WorkItemId})", + classId, candidate.Agent.Value, candidate.ModelId ?? "(default)", item.Id); + triedKeys.Add(key); + continue; + } + AgentCredential? candidateCredential; + try + { + candidateCredential = await ResolveAgentCredentialForInvocationAsync( + boundCandidateRunner, project, candidateTrialItem, ct); + } + catch (Exception credEx) when (credEx is not OperationCanceledException) + { + _log.LogWarning(credEx, + "Class '{ClassId}' member {Agent}/{Model} credential could not be resolved; refusing for fallback (work item {WorkItemId})", + classId, candidate.Agent.Value, candidate.ModelId ?? "(default)", item.Id); + triedKeys.Add(key); + continue; + } + var switchAssessment = AgentRunnerSwitchGate.AssessSwitch(boundCandidateRunner, candidateCredential); + if (!switchAssessment.Allowed) + { + _log.LogWarning( + "Class '{ClassId}' member {Agent}/{Model} refused for fallback (work item {WorkItemId}): {Reason}", + classId, candidate.Agent.Value, candidate.ModelId ?? "(default)", item.Id, + switchAssessment.RefusalReason ?? "credential cannot be materialised"); + triedKeys.Add(key); + continue; + } // The router's in-process exhausted-cache filters most stale // picks, but a member can be quota-failed in the persistent // observed-failure store (e.g. an earlier process recorded @@ -627,6 +688,10 @@ async Task MoveToNextMemberOrThrowAsync( continue; } nextMember = candidate; + // Assessed (and member-bound) above by the agent-switch + // credential gate; carried out so the swap below dispatches + // exactly the runner that was validated. + nextRunner = boundCandidateRunner; break; } @@ -692,11 +757,23 @@ await _fallbackHistory.RecordAsync(new AgentFallbackRecord( terminalException); } + // A terminal authentication failure with the post-swap guard + // still armed means the swapped-in agent 401d on its very + // first attempt: infrastructure (missing credential + // materialisation), not an agent re-authentication + // requirement. Without a preceding swap the guard is disarmed + // and genuine credential expiry still fails as auth-required. + if (terminalException is AgentAuthRequiredException terminalAuth && postSwapAuthGuardArmed) + { + postSwapAuthGuardArmed = false; + throw AgentRunnerSwitchGate.ToPostSwapInfrastructureFailure(terminalAuth, phase); + } + throw terminalException; } - if (!_agents.TryGet(nextMember.Agent, out var nextRunner)) - throw new InvalidOperationException($"No runner registered for fallback agent '{nextMember.Agent}'"); + if (nextRunner is null) + throw new InvalidOperationException($"No runner resolved for fallback agent '{nextMember.Agent}'"); if (quotaExhausted) { @@ -820,10 +897,23 @@ await _webhooks.PublishAsync(new WebhookEvent currentMember = nextMember; currentRunner = nextRunner; currentItem = trialItem; - // The fallback member brings its own configuration (e.g. a - // different Copilot BYOK provider or the native subscription), so - // the retry must run bound to the NEW member, not the exhausted one. - currentRunner = BindMemberRunner(currentRunner, currentMember); + // nextRunner was already bound to the NEW member by the + // agent-switch credential gate during candidate selection (the + // fallback member brings its own configuration, e.g. a different + // Copilot BYOK provider or the native subscription), so the retry + // runs bound to the incoming member, not the exhausted one. + // The swap arms the post-swap auth guard: a 401 on the very next + // attempt is infrastructure, not re-authentication. + postSwapAuthGuardArmed = true; + // The retry must execute with the incoming member's credential + // environment. Direct credential variables are baked into the + // sandbox spec at creation, so a warm reusable sandbox still + // carries the exhausted member's environment — the incoming + // agent's CLI would run without its own credentials and 401. + // Surrender it so the retry provisions a fresh sandbox from the + // incoming trial item's spec. No-op when reuse is disabled or no + // reusable sandbox is held; same-member retries never reach here. + await ReleaseAmbientWorkSandboxAsync(); } while (true) @@ -865,7 +955,13 @@ await MoveToNextMemberOrThrowAsync( try { - return await InvokeAttemptAsync(currentRunner, currentItem); + var attemptResult = await InvokeAttemptAsync(currentRunner, currentItem); + // A completed post-swap attempt (success or a failure the + // catches below do not convert) consumes the guard: later + // auth failures are genuine credential events, not swap + // artefacts. + postSwapAuthGuardArmed = false; + return attemptResult; } catch (TerminalQuotaError quotaEx) { @@ -888,6 +984,18 @@ await MoveToNextMemberOrThrowAsync( quotaResetAt: null, terminalException: authEx); } + catch (AgentAuthRequiredException authEx) when (postSwapAuthGuardArmed) + { + // The swapped-in runner 401d on its very first attempt. The + // swap was assessed as materialisable before dispatch, so + // this is infrastructure (the agent likely executed without + // its own credentials), not an agent re-authentication + // requirement. Without a preceding swap the guard is disarmed + // and the auth failure propagates unchanged, so a genuinely + // expired credential still fails the item as auth-required. + postSwapAuthGuardArmed = false; + throw AgentRunnerSwitchGate.ToPostSwapInfrastructureFailure(authEx, phase); + } catch (AgentAttemptTimeoutException timeoutEx) { var safeReason = SingleLineSummary(timeoutEx.Message); diff --git a/tests/CodeyBox.Tests/AgentRunnerSwitchTests.cs b/tests/CodeyBox.Tests/AgentRunnerSwitchTests.cs new file mode 100644 index 000000000..baac81a57 --- /dev/null +++ b/tests/CodeyBox.Tests/AgentRunnerSwitchTests.cs @@ -0,0 +1,773 @@ +using Microsoft.Extensions.Logging.Abstractions; +using CodeyBox.Agents; +using CodeyBox.Agents.Claude; +using CodeyBox.Agents.Cursor; +using CodeyBox.Audit; +using CodeyBox.Core; +using CodeyBox.Git; +using CodeyBox.Orchestrator; +using CodeyBox.Projects; +using CodeyBox.Sandbox; +using CodeyBox.Sandbox.Process; + +namespace CodeyBox.Tests; + +/// +/// Pins the mid-iteration agent-switch credential behaviour shared by the +/// quota-fallback path and the conflict-resolver path (work item +/// a09d2275): a phase entered under agent A and switched to agent B +/// mid-iteration must execute B with B's own credentials; a swap to an agent +/// whose credential cannot be materialised is refused with the original +/// failure kept and no dispatch attempted; and a provider 401 on the first +/// attempt after a swap is infrastructure, never "agent requires +/// re-authentication". +/// +public sealed class AgentRunnerSwitchTests : IDisposable +{ + private const string DirectTokenVar = "TEST_DIRECT_TOKEN"; + private const string CursorQuotaStderr = + "You're out of usage. Switch to Auto, or ask your admin to increase your limit to continue."; + + private readonly string _workspace; + + public AgentRunnerSwitchTests() => + _workspace = Directory.CreateTempSubdirectory("codeybox-switch-").FullName; + + public void Dispose() { try { Directory.Delete(_workspace, recursive: true); } catch { } } + + [Fact] + public void AssessSwitch_MismatchedCredentialAgent_IsRefused() + { + var runner = new CredentialCapturingRunner(AgentKind.Claude, new ScriptableAgent(AgentKind.Claude)); + var otherAgentCredential = new AgentCredential( + AgentKind.Cursor, + new Dictionary { [DirectTokenVar] = "cursor-key" }, + new Dictionary()); + + var assessment = AgentRunnerSwitchGate.AssessSwitch(runner, otherAgentCredential); + + Assert.False(assessment.Allowed); + Assert.Contains("cursor", assessment.RefusalReason, StringComparison.Ordinal); + Assert.Contains("claude", assessment.RefusalReason, StringComparison.Ordinal); + } + + [Fact] + public void AssessSwitch_UnclassifiableCredentialVariable_IsRefused() + { + var runner = new CredentialCapturingRunner(AgentKind.Claude, new ScriptableAgent(AgentKind.Claude)); + var unclassified = new AgentCredential( + AgentKind.Claude, + new Dictionary { ["UNCLASSIFIED_SWITCH_TOKEN"] = "secret" }, + new Dictionary()); + + var assessment = AgentRunnerSwitchGate.AssessSwitch(runner, unclassified); + + Assert.False(assessment.Allowed); + Assert.NotNull(assessment.RefusalReason); + Assert.DoesNotContain("secret", assessment.RefusalReason, StringComparison.Ordinal); + } + + [Fact] + public void AssessSwitch_MatchingClassifiedCredential_IsAllowed() + { + var runner = new CredentialCapturingRunner(AgentKind.Claude, new ScriptableAgent(AgentKind.Claude)); + var credential = new AgentCredential( + AgentKind.Claude, + new Dictionary { [DirectTokenVar] = "claude-key" }, + new Dictionary()); + + var assessment = AgentRunnerSwitchGate.AssessSwitch(runner, credential); + + Assert.True(assessment.Allowed); + Assert.Same(credential, assessment.Credential); + } + + [Fact] + public void AssessSwitch_NullCredential_IsAllowed() + { + // A missing credential is never a refusal: the candidate may still + // authenticate from ambient sandbox state, and a 401 that follows a + // swap is reclassified as infrastructure rather than refused here. + var runner = new CredentialCapturingRunner(AgentKind.Claude, new ScriptableAgent(AgentKind.Claude)); + + var assessment = AgentRunnerSwitchGate.AssessSwitch(runner, incomingCredential: null); + + Assert.True(assessment.Allowed); + Assert.Null(assessment.Credential); + } + + [Fact] + public void ToPostSwapInfrastructureFailure_PreservesAgentPhaseAndEvidence() + { + var auth = new AgentAuthRequiredException( + AgentKind.Copilot, + "rework", + "auth required from agent output: Authentication failed with provider (HTTP 401)."); + + var infra = AgentRunnerSwitchGate.ToPostSwapInfrastructureFailure(auth, "rework"); + + Assert.Equal(AgentKind.Copilot, infra.Agent); + Assert.Equal("rework", infra.Phase); + Assert.Contains("runner swap", infra.Message, StringComparison.Ordinal); + Assert.Contains("HTTP 401", infra.Message, StringComparison.Ordinal); + Assert.Same(auth, infra.InnerException); + } + + [Fact] + public async Task MidIterationSwap_ExecutesIncomingAgentWithItsOwnCredentials() + { + var seed = await TestSupport.CreateSeedRepoAsync(_workspace); + using var fix = BuildPipeline( + seed, + cursorCredential: CredentialFor(AgentKind.Cursor, "cursor-key"), + claudeCredential: CredentialFor(AgentKind.Claude, "claude-key")); + + fix.Cursor.ScriptedFailures.Enqueue(new AgentResult( + Success: false, + Summary: "agent exited 1", + Stdout: null, + Stderr: CursorQuotaStderr)); + fix.ClaudeAgent.WorkPlan.Enqueue(new FileWrite("ok.txt", "v1")); + + var item = NewItem(AgentKind.Cursor); + await fix.Store.CreateAsync(item); + await fix.Pipeline.RunAsync(item, CancellationToken.None); + + var final = await fix.Store.GetAsync(item.Id, CancellationToken.None); + Assert.NotNull(final); + Assert.Equal(WorkItemState.Done, final!.State); + + // The fallback ran and the incoming agent observed its OWN credential. + Assert.Equal(1, fix.Claude.CallCount); + var seen = Assert.Single(fix.Claude.SeenCredentials); + Assert.NotNull(seen); + Assert.Equal(AgentKind.Claude, seen!.Agent); + Assert.Equal("claude-key", seen.EnvironmentVariables[DirectTokenVar]); + + // End to end: each attempt's sandbox spec carried THAT attempt's + // credential — the exhausted agent's key on the first spec, the + // incoming agent's key on the second. Before the swap released the + // warm sandbox, the second spec never existed and the incoming agent + // inherited the first sandbox's environment. + Assert.Equal(2, fix.Sandboxes.Specs.Count); + Assert.Single( + fix.Sandboxes.Specs, + s => s.Environment.TryGetValue(DirectTokenVar, out var v) && v == "cursor-key"); + Assert.Single( + fix.Sandboxes.Specs, + s => s.Environment.TryGetValue(DirectTokenVar, out var v) && v == "claude-key"); + } + + [Fact] + public async Task MidIterationSwap_UnmaterialisableCredential_RefusedWithOriginalFailure() + { + var seed = await TestSupport.CreateSeedRepoAsync(_workspace); + using var fix = BuildPipeline( + seed, + cursorCredential: CredentialFor(AgentKind.Cursor, "cursor-key"), + // Belongs to cursor, not claude: the switch gate must refuse it. + claudeCredential: new AgentCredential( + AgentKind.Cursor, + new Dictionary { [DirectTokenVar] = "cursor-key" }, + new Dictionary())); + fix.ClaudeAgent.WorkPlan.Enqueue(new FileWrite("must-not-run.txt", "v1")); + + fix.Cursor.ScriptedFailures.Enqueue(new AgentResult( + Success: false, + Summary: "agent exited 1", + Stdout: null, + Stderr: CursorQuotaStderr)); + + var item = NewItem(AgentKind.Cursor); + await fix.Store.CreateAsync(item); + await fix.Pipeline.RunAsync(item, CancellationToken.None); + + var final = await fix.Store.GetAsync(item.Id, CancellationToken.None); + Assert.NotNull(final); + // The original quota failure is preserved — the item parks on the + // cursor quota failure (quota park, or the pre-existing healthy-probe + // redirect to transient retry that also applies to the established + // "no registered runner" skip path), never on an auth failure from a + // doomed dispatch of the refused agent. + Assert.True( + final!.State == WorkItemState.WaitingForQuotaReset + || final.State == WorkItemState.WaitingForTransientRetry, + $"unexpected state {final.State}"); + Assert.Contains("exhausted mid-work", final.LastError ?? string.Empty, StringComparison.Ordinal); + Assert.Contains("quota failure", final.LastError ?? string.Empty, StringComparison.Ordinal); + Assert.NotEqual(WorkItemFailureKinds.AuthRequired, final.FailureKind); + + // No dispatch of the refused agent was attempted. + Assert.Equal(0, fix.Claude.CallCount); + Assert.Empty(fix.Claude.SeenCredentials); + Assert.DoesNotContain( + fix.Webhooks.Events, + e => e.Event == "agent.fallback" + && e.Details is AgentFallbackDetails d + && d.ToAgent == AgentKind.Claude.Value); + } + + [Fact] + public async Task PostSwapAuthFailure_IsInfrastructure_NotReauthentication() + { + var seed = await TestSupport.CreateSeedRepoAsync(_workspace); + using var fix = BuildPipeline( + seed, + cursorCredential: CredentialFor(AgentKind.Cursor, "cursor-key"), + claudeCredential: CredentialFor(AgentKind.Claude, "claude-key")); + + fix.Cursor.ScriptedFailures.Enqueue(new AgentResult( + Success: false, + Summary: "agent exited 1", + Stdout: null, + Stderr: CursorQuotaStderr)); + // The swapped-in agent 401s on its very first attempt — the shape the + // detector used to report as "agent requires re-authentication". + fix.ClaudeAgent.ScriptedExceptions.Enqueue(new AgentAuthRequiredException( + AgentKind.Claude, + "work", + "auth required from agent output: Authentication failed with provider at https://openrouter.ai/api/v1 (HTTP 401).")); + + var item = NewItem(AgentKind.Cursor); + await fix.Store.CreateAsync(item); + await fix.Pipeline.RunAsync(item, CancellationToken.None); + + var final = await fix.Store.GetAsync(item.Id, CancellationToken.None); + Assert.NotNull(final); + Assert.Equal(WorkItemState.Failed, final!.State); + Assert.Equal(WorkItemFailureKinds.Infrastructure, final!.FailureKind); + Assert.NotEqual(WorkItemFailureKinds.AuthRequired, final.FailureKind); + Assert.Contains("runner swap", final.LastError ?? string.Empty, StringComparison.Ordinal); + Assert.Contains("HTTP 401", final.LastError ?? string.Empty, StringComparison.Ordinal); + + // The swap itself happened exactly once — the failure is the + // classification of the post-swap attempt, not a missing dispatch. + Assert.Equal(1, fix.Claude.CallCount); + } + + [Fact] + public async Task AuthFailureWithoutSwap_RemainsReauthenticationRequirement() + { + // A genuinely expired credential with no preceding swap must still + // fail the item as auth-required: the post-swap guard must not weaken + // the auth-failure detector. + var seed = await TestSupport.CreateSeedRepoAsync(_workspace); + using var fix = BuildSoloPipeline( + seed, + cursorCredential: CredentialFor(AgentKind.Cursor, "cursor-key")); + + fix.Cursor.ScriptedExceptions.Enqueue(new AgentAuthRequiredException( + AgentKind.Cursor, + "work", + "auth required from agent output: login prompt matched.")); + + var item = NewItem(AgentKind.Cursor); + await fix.Store.CreateAsync(item); + await fix.Pipeline.RunAsync(item, CancellationToken.None); + + var final = await fix.Store.GetAsync(item.Id, CancellationToken.None); + Assert.NotNull(final); + Assert.Equal(WorkItemState.Failed, final!.State); + Assert.Equal(WorkItemFailureKinds.AuthRequired, final!.FailureKind); + } + + [Fact] + public async Task ConflictResolver_RefusedCandidateNeverDispatches_AndWinnerScopedToOwnCredential() + { + var sandbox = new SwitchConflictSandbox(); + sandbox.AddConflictedFile("src/a.txt", "<<<<<<<\nbase\n=======\nwork\n>>>>>>>\n"); + + var refused = new EnvRecordingRunner(new AgentKind("refused-agent")); + var refusedCredential = new AgentCredential( + new AgentKind("other-agent"), + new Dictionary { ["REFUSED_TOKEN"] = "refused-secret" }, + new Dictionary()); + + var winner = new EnvRecordingRunner(new AgentKind("winner-agent")); + var winnerCredential = new AgentCredential( + winner.Kind, + new Dictionary { ["WINNER_TOKEN"] = "winner-secret" }, + new Dictionary()); + + var observing = new EnvObservingSandbox(sandbox); + var resolver = new AgenticConflictResolver( + new AgenticConflictResolverOptionsSnapshot(new AgenticConflictResolverOptions + { + MaxIterations = 2, + MaxAttemptsPerAgent = 1, + })); + + var result = await resolver.ResolveAsync( + observing, + "/work", + WorkItemId.New(), + new AgenticConflictResolverContext("main", "feature", AgenticConflictResolverOperation.Rebase), + [ + new AgenticConflictResolverCandidate(refused, refusedCredential), + new AgenticConflictResolverCandidate(winner, winnerCredential), + ], + CancellationToken.None); + + Assert.True(result.Success, result.Summary); + Assert.Same(winner, result.ChosenRunner); + Assert.Equal(0, refused.InvocationCount); + Assert.Equal(1, winner.InvocationCount); + + // The refused candidate never dispatched, and its secret never + // reached any exec environment; the winner ran scoped to its own + // credential. (Refusals are recorded on the failure trail; success + // summaries only name the winning attempt.) + var seenWinner = Assert.Single(winner.SeenCredentials); + Assert.Same(winnerCredential, seenWinner); + Assert.DoesNotContain( + observing.SeenEnvironments.SelectMany(env => env.Keys), + name => name == "REFUSED_TOKEN"); + Assert.Contains( + observing.SeenEnvironments, + env => env.TryGetValue("WINNER_TOKEN", out var v) && v == "winner-secret"); + } + + [Fact] + public async Task ConflictResolver_AllCandidatesRefused_FailsWithRefusalTrail() + { + var sandbox = new SwitchConflictSandbox(); + sandbox.AddConflictedFile("src/a.txt", "<<<<<<<\nbase\n=======\nwork\n>>>>>>>\n"); + + var refused = new EnvRecordingRunner(new AgentKind("refused-agent")); + var refusedCredential = new AgentCredential( + new AgentKind("other-agent"), + new Dictionary { ["REFUSED_TOKEN"] = "refused-secret" }, + new Dictionary()); + + var resolver = new AgenticConflictResolver( + new AgenticConflictResolverOptionsSnapshot(new AgenticConflictResolverOptions + { + MaxIterations = 2, + MaxAttemptsPerAgent = 1, + })); + + var result = await resolver.ResolveAsync( + sandbox, + "/work", + WorkItemId.New(), + new AgenticConflictResolverContext("main", "feature", AgenticConflictResolverOperation.Rebase), + [new AgenticConflictResolverCandidate(refused, refusedCredential)], + CancellationToken.None); + + Assert.False(result.Success); + Assert.Equal(0, refused.InvocationCount); + Assert.Null(result.ChosenRunner); + // The refusal is visible on the failure trail instead of dispatching + // a candidate whose credential cannot be materialised. + Assert.Contains("credential refused", result.Summary, StringComparison.Ordinal); + Assert.Contains("refused-agent", result.Summary, StringComparison.Ordinal); + } + + // ── Harness ────────────────────────────────────────────────────────────── + + private static AgentCredential CredentialFor(AgentKind kind, string token) => + new(kind, + new Dictionary { [DirectTokenVar] = token }, + new Dictionary()); + + private SwitchFixture BuildPipeline( + string seedRepoUrl, + AgentCredential? cursorCredential, + AgentCredential? claudeCredential) + { + var gitRoot = Path.Combine(_workspace, "repos-" + Guid.NewGuid().ToString("N")[..8]); + var stateDb = Path.Combine(_workspace, "state-" + Guid.NewGuid().ToString("N")[..8] + ".db"); + + var store = new SqliteWorkItemStore(stateDb); + var gitHost = new LocalGitHost(new LocalGitHostOptions { RootDirectory = gitRoot }, NullLogger.Instance); + var sandboxes = new SwitchRecordingSandboxProvider( + new ProcessSandboxProvider(NullLogger.Instance)); + var prs = new InMemoryPullRequestService(); + var webhooks = new CapturingWebhookDispatcher(); + + var cursorInner = new ScriptableAgent(AgentKind.Cursor); + var claudeInner = new ScriptableAgent(AgentKind.Claude); + var cursor = new CredentialCapturingRunner(AgentKind.Cursor, cursorInner); + var claude = new CredentialCapturingRunner(AgentKind.Claude, claudeInner); + var registry = new AgentRegistry([cursor, claude]); + + var frontier = new AgentClass + { + Id = "frontier", + DisplayName = "Frontier", + Members = + [ + new AgentMembership { Agent = AgentKind.Cursor, Billing = AgentBilling.Subscription, QualityScore = 100 }, + new AgentMembership { Agent = AgentKind.Claude, Billing = AgentBilling.Subscription, QualityScore = 100 }, + ], + }; + + var project = new Project + { + Id = new ProjectId("test-project"), + DisplayName = "Test", + RepositoryUrl = seedRepoUrl, + DefaultBaseBranch = "main", + DefaultAgent = AgentKind.Cursor, + DefaultAgentClass = "frontier", + Audit = new ProjectAudit { MaxIterations = 1, AuditTypes = [] }, + }; + + var projects = new InMemoryProjectRepository(project); + var composer = new ProjectAuditorComposer(new ScriptedAuditorCatalog([])); + + var cursorProbe = new RecordingProbe(AgentKind.Cursor); + var claudeProbe = new RecordingProbe(AgentKind.Claude); + + var quotaOptions = new QuotaRouterOptions { MinQuotaPct = 10.0 }; + var router = new AgentClassRouter( + [frontier], + [cursorProbe, claudeProbe], + quotaOptions, + NullLogger.Instance); + + var fallbackHistory = new InMemoryAgentFallbackHistoryStore(); + var terminalTransitions = TestSupport.CreateTerminalTransition(store, webhooks, projects); + + var pipeline = new PipelineRunner( + sandboxes, gitHost, registry, + new PerAgentCredentialProvider(new Dictionary + { + [AgentKind.Cursor] = cursorCredential, + [AgentKind.Claude] = claudeCredential, + }), + prs, projects, new TestUpstreamFactory(), composer, + store, webhooks, + new PipelineOptions { SandboxImageReference = "ignored", AgentAllowedHosts = [] }, + NullLogger.Instance, + auditQuotaProbes: [cursorProbe, claudeProbe], + auditQuotaOptions: quotaOptions, + classRouter: router, + fallbackHistory: fallbackHistory, + quotaClassifier: new CompositeQuotaFailureClassifier(new IAgentQuotaFailureDetector[] + { + new CursorQuotaFailureDetector(), + new ClaudeQuotaFailureDetector(), + }), + requiredBuildVerifier: TestRequiredBuildVerifier.NotApplicable, + terminalTransitions: terminalTransitions, + terminalRevisionBuilder: terminalTransitions); + + return new SwitchFixture(pipeline, store, cursorInner, cursor, claudeInner, claude, webhooks, sandboxes); + } + + private SwitchFixture BuildSoloPipeline(string seedRepoUrl, AgentCredential? cursorCredential) + { + var fixture = BuildPipeline(seedRepoUrl, cursorCredential, claudeCredential: null); + return fixture; + } + + private static WorkItem NewItem(AgentKind initialAgent) => new() + { + Id = WorkItemId.New(), + ProjectId = new ProjectId("test-project"), + Title = "agent switch test", + Prompt = "do thing", + BaseBranch = "main", + Agent = initialAgent, + AgentClassId = null, + PushUpstream = false, + }; + + private sealed class SwitchFixture : IDisposable + { + public PipelineRunner Pipeline { get; } + public SqliteWorkItemStore Store { get; } + public ScriptableAgent Cursor { get; } + public CredentialCapturingRunner CursorRunner { get; } + public ScriptableAgent ClaudeAgent { get; } + public CredentialCapturingRunner Claude { get; } + public CapturingWebhookDispatcher Webhooks { get; } + public SwitchRecordingSandboxProvider Sandboxes { get; } + + public SwitchFixture( + PipelineRunner pipeline, + SqliteWorkItemStore store, + ScriptableAgent cursor, + CredentialCapturingRunner cursorRunner, + ScriptableAgent claudeAgent, + CredentialCapturingRunner claude, + CapturingWebhookDispatcher webhooks, + SwitchRecordingSandboxProvider sandboxes) + { + Pipeline = pipeline; + Store = store; + Cursor = cursor; + CursorRunner = cursorRunner; + ClaudeAgent = claudeAgent; + Claude = claude; + Webhooks = webhooks; + Sandboxes = sandboxes; + } + + public void Dispose() => Store.Dispose(); + } + + private sealed class PerAgentCredentialProvider( + IReadOnlyDictionary byAgent) : ICredentialProvider + { + public Task GetAsync(AgentKind agent, CancellationToken ct = default) => + Task.FromResult(byAgent.TryGetValue(agent, out var credential) ? credential : null); + } + + private sealed class SwitchRecordingSandboxProvider(ISandboxProvider inner) : ISandboxProvider + { + private readonly List _specs = new(); + + public string Name => inner.Name; + + public IReadOnlyList Specs + { + get { lock (_specs) return _specs.ToList(); } + } + + public Task CreateAsync(SandboxSpec spec, CancellationToken ct = default) + { + lock (_specs) _specs.Add(spec); + return inner.CreateAsync(spec, ct); + } + + public Task> ListAllManagedAsync(CancellationToken ct) + => inner.ListAllManagedAsync(ct); + + public Task DisposeLeakedAsync(string name, CancellationToken ct) + => inner.DisposeLeakedAsync(name, ct); + } + + /// + /// Policy-declaring test runner that captures the credential it was + /// invoked with and delegates execution to an inner + /// so quota-fallback scenarios script + /// failures and file writes exactly like the existing fallback tests. + /// + private sealed class CredentialCapturingRunner(AgentKind kind, ScriptableAgent inner) + : IAgentRunner, IAgentCredentialEnvironmentPolicy + { + public AgentKind Kind { get; } = kind; + public ScriptableAgent Inner { get; } = inner; + public List SeenCredentials { get; } = []; + public int CallCount => Inner.CallCount; + + public IReadOnlySet DirectCredentialEnvironmentVariables { get; } = + new HashSet(StringComparer.Ordinal) { DirectTokenVar }; + + public IReadOnlySet FileBackedCredentialEnvironmentVariables { get; } = + new HashSet(StringComparer.Ordinal); + + public IReadOnlyList CredentialFileDestinations { get; } = []; + + public Task RunAsync( + ISandbox sandbox, + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null, + CancellationToken ct = default, + Action? stdoutChunkCallback = null, + bool captureStructuredStream = false) + { + SeenCredentials.Add(credential); + return Inner.RunAsync( + sandbox, workingDirectory, prompt, credential, + modelId, reasoningMode, ct, stdoutChunkCallback, captureStructuredStream); + } + } + + /// + /// Minimal in-memory sandbox for the resolver-path test: serves unmerged + /// paths, marker grep, and git add through only so + /// the candidate's scoped environment is observable. + /// + private sealed class SwitchConflictSandbox : ISandbox + { + private readonly HashSet _unmerged = new(StringComparer.Ordinal); + private readonly Dictionary _files = new(StringComparer.Ordinal); + + public string Id => "switch-conflict-fake"; + public List AddedFiles { get; } = new(); + + public void AddConflictedFile(string relativePath, string content) + { + _files[relativePath] = content; + _unmerged.Add(relativePath); + } + + public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + var argv = exec.Argv; + if (argv.Count >= 4 && argv[0] == "sh" && argv[1] == "-c" + && argv[2].StartsWith("cat > ", StringComparison.Ordinal)) + { + // Scripted file write: ["sh", "-c", "cat > \"$0\"", "/"] with Stdin payload. + var target = argv.Count > 3 ? argv[3] : string.Empty; + const string workPrefix = "/work/"; + if (!target.StartsWith(workPrefix, StringComparison.Ordinal)) + return Task.FromResult(new SandboxExecResult(1, "", $"unsupported write target '{target}'")); + _files[target[workPrefix.Length..]] = exec.Stdin ?? string.Empty; + return Task.FromResult(new SandboxExecResult(0, "", "")); + } + + if (argv.Count >= 4 && argv[0] == "git" && argv[1] == "-C" && argv[3] == "ls-files" && argv.Contains("-u")) + { + var sb = new System.Text.StringBuilder(); + var oid = new string('a', 40); + foreach (var path in _unmerged.Order(StringComparer.Ordinal)) + sb.Append("100644 ").Append(oid).Append(" 2\t").Append(path).Append('\0'); + return Task.FromResult(new SandboxExecResult(0, sb.ToString(), "")); + } + + if (argv.Count >= 4 && argv[0] == "git" && argv[1] == "-C" && argv[3] == "diff" + && argv.Contains("--diff-filter=U")) + { + return Task.FromResult(new SandboxExecResult(0, string.Join('\n', _unmerged.Order(StringComparer.Ordinal)), "")); + } + + if (argv.Count >= 4 && argv[0] == "git" && argv[1] == "-C" && argv[3] == "grep") + { + var sepIdx = -1; + for (var i = 4; i < argv.Count; i++) + { + if (argv[i] == "--") { sepIdx = i; break; } + } + var matched = new List(); + if (sepIdx >= 0) + { + for (var i = sepIdx + 1; i < argv.Count; i++) + { + var path = argv[i]; + if (_files.TryGetValue(path, out var content) && HasMarkers(content)) + matched.Add(path); + } + } + return matched.Count == 0 + ? Task.FromResult(new SandboxExecResult(1, "", "")) + : Task.FromResult(new SandboxExecResult(0, string.Join('\n', matched), "")); + } + + if (argv.Count >= 5 && argv[0] == "git" && argv[1] == "-C" && argv[3] == "add" && argv[4] == "--") + { + for (var i = 5; i < argv.Count; i++) + { + if (!_files.ContainsKey(argv[i])) + return Task.FromResult(new SandboxExecResult(1, "", $"pathspec '{argv[i]}' did not match")); + AddedFiles.Add(argv[i]); + _unmerged.Remove(argv[i]); + } + return Task.FromResult(new SandboxExecResult(0, "", "")); + } + + return Task.FromResult(new SandboxExecResult(0, "", "")); + } + + public Task KillActiveExecsAsync(CancellationToken ct = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private static bool HasMarkers(string content) + { + foreach (var line in content.Split('\n')) + { + if (line.StartsWith("<<<<<<<", StringComparison.Ordinal) + || line.StartsWith("=======", StringComparison.Ordinal) + || line.StartsWith(">>>>>>>", StringComparison.Ordinal)) + { + return true; + } + } + return false; + } + } + + private sealed class EnvObservingSandbox(ISandbox inner) : ISandboxDecorator + { + private readonly List> _seen = new(); + + public ISandbox InnerSandbox => inner; + public string Id => inner.Id; + public SandboxAgentOutputTransportKind AgentOutputTransportKind => inner.AgentOutputTransportKind; + public SandboxBatchLaunchMode BatchLaunchMode => inner.BatchLaunchMode; + public SandboxResourceMetrics? ResourceMetrics => inner.ResourceMetrics; + + public IReadOnlyList> SeenEnvironments + { + get { lock (_seen) return _seen.ToList(); } + } + + public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + lock (_seen) + _seen.Add(exec.ExtraEnvironment ?? new Dictionary(StringComparer.Ordinal)); + return inner.ExecAsync(exec, ct); + } + + public Task SyncStateToHostAsync(CancellationToken ct = default) => inner.SyncStateToHostAsync(ct); + public Task KillActiveExecsAsync(CancellationToken ct = default) => inner.KillActiveExecsAsync(ct); + public Task GetScreenshotAsync(CancellationToken ct = default) => inner.GetScreenshotAsync(ct); + + public Task SynthesizeInputAsync(IReadOnlyList events, CancellationToken ct = default) => + inner.SynthesizeInputAsync(events, ct); + + public Task GetAccessibilityAtPointAsync(int x, int y, CancellationToken ct = default) => + inner.GetAccessibilityAtPointAsync(x, y, ct); + + public Task GetAccessibilityTreeJsonAsync(CancellationToken ct = default) => + inner.GetAccessibilityTreeJsonAsync(ct); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + /// + /// Policy-declaring resolver-path runner that resolves the conflict + /// through execs (so the scoped environment + /// applies) and records the credential it ran with. + /// + private sealed class EnvRecordingRunner(AgentKind kind) : IAgentRunner, IAgentCredentialEnvironmentPolicy + { + public AgentKind Kind { get; } = kind; + public int InvocationCount { get; private set; } + public List SeenCredentials { get; } = []; + + public IReadOnlySet DirectCredentialEnvironmentVariables { get; } = + new HashSet(StringComparer.Ordinal) { "WINNER_TOKEN", "REFUSED_TOKEN" }; + + public IReadOnlySet FileBackedCredentialEnvironmentVariables { get; } = + new HashSet(StringComparer.Ordinal); + + public IReadOnlyList CredentialFileDestinations { get; } = []; + + public async Task RunAsync( + ISandbox sandbox, + string workingDirectory, + string prompt, + AgentCredential? credential, + string? modelId = null, + string? reasoningMode = null, + CancellationToken ct = default, + Action? stdoutChunkCallback = null, + bool captureStructuredStream = false) + { + InvocationCount++; + SeenCredentials.Add(credential); + var write = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["sh", "-c", "cat > \"$0\"", $"{workingDirectory}/src/a.txt"], + Stdin = "base + work\n", + }, ct); + if (!write.Success) + return new AgentResult(false, "write failed", write.Stdout, write.Stderr); + var add = await sandbox.ExecAsync(new SandboxExec + { + Argv = ["git", "-C", workingDirectory, "add", "--", "src/a.txt"], + }, ct); + return add.Success + ? new AgentResult(true, "resolved", null, null) + : new AgentResult(false, "git add failed", add.Stdout, add.Stderr); + } + } +}