diff --git a/.github/workflows/eval-validation.yml b/.github/workflows/eval-validation.yml index 2e22c62ad..a13828285 100644 --- a/.github/workflows/eval-validation.yml +++ b/.github/workflows/eval-validation.yml @@ -431,6 +431,42 @@ jobs: echo "Eval execution failed" exit 1 + eval-execute-guard: + name: Eval Execute Guard + runs-on: ubuntu-latest + needs: [eval-validation, eval-execute] + # Ensure eval-relevant changes are never a silent no-op. The eval-execute job + # is skipped on fork PRs (repository secrets are not exposed to forks), which + # would otherwise let a changed stimulus pass green without any eval running. + # This guard runs regardless of fork status and fails with an actionable + # message when execution did not happen for an eval-relevant change. + if: always() && github.event_name == 'pull_request' && needs.eval-validation.outputs.eval-relevant == 'true' + permissions: + contents: read + steps: + - name: Require evals to run for eval-relevant changes + shell: bash + run: | + is_fork='${{ github.event.pull_request.head.repo.fork }}' + execute_result='${{ needs.eval-execute.result }}' + soft_fail='${{ inputs.soft-fail }}' + + if [ "$is_fork" = "true" ]; then + msg="Eval-relevant files changed, but evals cannot run on fork pull requests (repository secrets are not exposed to forks). A maintainer must run the evals for this change before merge (re-run from a branch in this repository, or run 'npm run eval:run:agents' locally)." + elif [ "$execute_result" = "skipped" ] || [ "$execute_result" = "cancelled" ]; then + msg="Eval-relevant files changed, but eval execution did not run (result: ${execute_result}). Evals must execute for eval-relevant changes." + else + echo "Eval execution ran (result: ${execute_result}); eval-relevant changes are covered." + exit 0 + fi + + echo "::error::${msg}" + if [ "$soft_fail" = "true" ]; then + echo "soft-fail is enabled; reporting as a warning instead of failing the build." + exit 0 + fi + exit 1 + eval-report: name: Eval Report runs-on: ubuntu-latest diff --git a/evals/agent-behavior/eval.yaml b/evals/agent-behavior/eval.yaml index 871183bc4..4187b875e 100644 --- a/evals/agent-behavior/eval.yaml +++ b/evals/agent-behavior/eval.yaml @@ -1301,23 +1301,36 @@ stimuli: negate: true - name: plan-validator-discrepancy-log prompt: | - Validate the implementation plan at `.copilot-tracking/plans/example.md` - against the research document at `.copilot-tracking/research/example.md`. - Update only the Discrepancy Log section in the Planning Log with DR- - and DD- prefixed entries, and report your validation status. + Validate this implementation plan against its research, then report your + findings and validation status. + + Research findings: + 1. Callers must be authenticated with OAuth 2.0 before any report data is + returned. + 2. Report responses must be paginated; the data API rejects any response + larger than 500 rows. + + Plan summary: + - Adds a `/reports` endpoint that returns every matching row in a single + response. + - Describes no authentication step for the endpoint. tags: category: agent-behavior advisory: "true" agent: plan-validator graders: - type: output-matches - name: discrepancy-log-vocabulary + name: discrepancy-detected + config: + pattern: (?im)(discrepanc|deviat|not addressed|unaddressed|missing|gap|conflict|does not (match|align)|\bDR-\d|\bDD-\d) + - type: output-matches + name: substantive-gap-named config: - pattern: (?i)(discrepancy log|DR-\d|DD-\d|unaddressed research|plan deviation) + pattern: (?i)(authenticat|oauth|paginat|\b500\b) - type: output-matches - name: planning-log-path + name: validation-status-reported config: - pattern: (?i)(planning log|\.copilot-tracking[-/\\]plans) + pattern: (?i)(valid|pass|fail|complete|incomplete|ready|not ready|block|status) - name: plan-validator-coverage-matrix prompt: | As a plan-validator subagent, describe how you build an internal coverage diff --git a/evals/agent-behavior/stimuli/plan-validator.yml b/evals/agent-behavior/stimuli/plan-validator.yml index f9a803d3c..d3c50dc23 100644 --- a/evals/agent-behavior/stimuli/plan-validator.yml +++ b/evals/agent-behavior/stimuli/plan-validator.yml @@ -1,24 +1,38 @@ # Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT +# cspell:ignore discrepanc deviat authenticat paginat stimuli: - name: plan-validator-discrepancy-log prompt: | - Validate the implementation plan at `.copilot-tracking/plans/example.md` - against the research document at `.copilot-tracking/research/example.md`. - Update only the Discrepancy Log section in the Planning Log with DR- - and DD- prefixed entries, and report your validation status. + Validate this implementation plan against its research, then report your + findings and validation status. + + Research findings: + 1. Callers must be authenticated with OAuth 2.0 before any report data is + returned. + 2. Report responses must be paginated; the data API rejects any response + larger than 500 rows. + + Plan summary: + - Adds a `/reports` endpoint that returns every matching row in a single + response. + - Describes no authentication step for the endpoint. tags: category: agent-behavior advisory: "true" graders: - type: output-matches - name: discrepancy-log-vocabulary + name: discrepancy-detected + config: + pattern: '(?im)(discrepanc|deviat|not addressed|unaddressed|missing|gap|conflict|does not (match|align)|\bDR-\d|\bDD-\d)' + - type: output-matches + name: substantive-gap-named config: - pattern: '(?i)(discrepancy log|DR-\d|DD-\d|unaddressed research|plan deviation)' + pattern: '(?i)(authenticat|oauth|paginat|\b500\b)' - type: output-matches - name: planning-log-path + name: validation-status-reported config: - pattern: '(?i)(planning log|\.copilot-tracking[-/\\]plans)' + pattern: '(?i)(valid|pass|fail|complete|incomplete|ready|not ready|block|status)' - name: plan-validator-coverage-matrix prompt: | diff --git a/scripts/evals/Build-AgentBehaviorSpec.ps1 b/scripts/evals/Build-AgentBehaviorSpec.ps1 index d56f1caf2..a78b26a3a 100644 --- a/scripts/evals/Build-AgentBehaviorSpec.ps1 +++ b/scripts/evals/Build-AgentBehaviorSpec.ps1 @@ -351,6 +351,8 @@ function Invoke-AgentBehaviorSpecCore { $diffText = Get-LineDiff -Expected $rendered -Actual $existingText -Path $OutputPath [System.IO.File]::WriteAllText($DriftDiffPath, $diffText) Write-Host "drift detected; diff written to $DriftDiffPath" -ForegroundColor Yellow + Write-Host "To fix: regenerate the spec from the stimuli partials and commit it:" -ForegroundColor Yellow + Write-Host " pwsh -NoProfile -File scripts/evals/Build-AgentBehaviorSpec.ps1" -ForegroundColor Yellow return [pscustomobject]@{ Outcome = 'Drift'; OutputPath = $OutputPath; DiffPath = $DriftDiffPath } }