test(evals): harden plan-validator discrepancy stimulus against prompt echo

.github/workflows/eval-validation.yml

@@ -431,6 +431,42 @@ jobs:
           echo "Eval execution failed"
           exit 1
 
+  eval-execute-guard:
+    name: Eval Execute Guard
+    runs-on: ubuntu-latest
+    needs: [eval-validation, eval-execute]
+    # Ensure eval-relevant changes are never a silent no-op. The eval-execute job
+    # is skipped on fork PRs (repository secrets are not exposed to forks), which
+    # would otherwise let a changed stimulus pass green without any eval running.
+    # This guard runs regardless of fork status and fails with an actionable
+    # message when execution did not happen for an eval-relevant change.
+    if: always() && github.event_name == 'pull_request' && needs.eval-validation.outputs.eval-relevant == 'true'
+    permissions:
+      contents: read
+    steps:
+      - name: Require evals to run for eval-relevant changes
+        shell: bash
+        run: |
+          is_fork='${{ github.event.pull_request.head.repo.fork }}'
+          execute_result='${{ needs.eval-execute.result }}'
+          soft_fail='${{ inputs.soft-fail }}'
+
+          if [ "$is_fork" = "true" ]; then
+            msg="Eval-relevant files changed, but evals cannot run on fork pull requests (repository secrets are not exposed to forks). A maintainer must run the evals for this change before merge (re-run from a branch in this repository, or run 'npm run eval:run:agents' locally)."
+          elif [ "$execute_result" = "skipped" ] || [ "$execute_result" = "cancelled" ]; then
+            msg="Eval-relevant files changed, but eval execution did not run (result: ${execute_result}). Evals must execute for eval-relevant changes."
+          else
+            echo "Eval execution ran (result: ${execute_result}); eval-relevant changes are covered."
+            exit 0
+          fi
+
+          echo "::error::${msg}"
+          if [ "$soft_fail" = "true" ]; then
+            echo "soft-fail is enabled; reporting as a warning instead of failing the build."
+            exit 0
+          fi
+          exit 1
+
   eval-report:
     name: Eval Report
     runs-on: ubuntu-latest

evals/agent-behavior/eval.yaml

@@ -1301,23 +1301,36 @@ stimuli:
       negate: true
 - name: plan-validator-discrepancy-log
   prompt: |
-    Validate the implementation plan at `.copilot-tracking/plans/example.md`
-    against the research document at `.copilot-tracking/research/example.md`.
-    Update only the Discrepancy Log section in the Planning Log with DR-
-    and DD- prefixed entries, and report your validation status.
+    Validate this implementation plan against its research, then report your
+    findings and validation status.
+
+    Research findings:
+    1. Callers must be authenticated with OAuth 2.0 before any report data is
+       returned.
+    2. Report responses must be paginated; the data API rejects any response
+       larger than 500 rows.
+
+    Plan summary:
+    - Adds a `/reports` endpoint that returns every matching row in a single
+      response.
+    - Describes no authentication step for the endpoint.
   tags:
     category: agent-behavior
     advisory: "true"
     agent: plan-validator
   graders:
   - type: output-matches
-    name: discrepancy-log-vocabulary
+    name: discrepancy-detected
+    config:
+      pattern: (?im)(discrepanc|deviat|not addressed|unaddressed|missing|gap|conflict|does not (match|align)|\bDR-\d|\bDD-\d)
+  - type: output-matches
+    name: substantive-gap-named
     config:
-      pattern: (?i)(discrepancy log|DR-\d|DD-\d|unaddressed research|plan deviation)
+      pattern: (?i)(authenticat|oauth|paginat|\b500\b)
   - type: output-matches
-    name: planning-log-path
+    name: validation-status-reported
     config:
-      pattern: (?i)(planning log|\.copilot-tracking[-/\\]plans)
+      pattern: (?i)(valid|pass|fail|complete|incomplete|ready|not ready|block|status)
 - name: plan-validator-coverage-matrix
   prompt: |
     As a plan-validator subagent, describe how you build an internal coverage

evals/agent-behavior/stimuli/plan-validator.yml

@@ -1,24 +1,38 @@
 # Copyright (c) 2026 Microsoft Corporation. All rights reserved.
 # SPDX-License-Identifier: MIT
+# cspell:ignore discrepanc deviat authenticat paginat
 stimuli:
   - name: plan-validator-discrepancy-log
     prompt: |
-      Validate the implementation plan at `.copilot-tracking/plans/example.md`
-      against the research document at `.copilot-tracking/research/example.md`.
-      Update only the Discrepancy Log section in the Planning Log with DR-
-      and DD- prefixed entries, and report your validation status.
+      Validate this implementation plan against its research, then report your
+      findings and validation status.
+
+      Research findings:
+      1. Callers must be authenticated with OAuth 2.0 before any report data is
+         returned.
+      2. Report responses must be paginated; the data API rejects any response
+         larger than 500 rows.
+
+      Plan summary:
+      - Adds a `/reports` endpoint that returns every matching row in a single
+        response.
+      - Describes no authentication step for the endpoint.
     tags:
       category: agent-behavior
       advisory: "true"
     graders:
       - type: output-matches
-        name: discrepancy-log-vocabulary
+        name: discrepancy-detected
+        config:
+          pattern: '(?im)(discrepanc|deviat|not addressed|unaddressed|missing|gap|conflict|does not (match|align)|\bDR-\d|\bDD-\d)'
+      - type: output-matches
+        name: substantive-gap-named
         config:
-          pattern: '(?i)(discrepancy log|DR-\d|DD-\d|unaddressed research|plan deviation)'
+          pattern: '(?i)(authenticat|oauth|paginat|\b500\b)'
       - type: output-matches
-        name: planning-log-path
+        name: validation-status-reported
         config:
-          pattern: '(?i)(planning log|\.copilot-tracking[-/\\]plans)'
+          pattern: '(?i)(valid|pass|fail|complete|incomplete|ready|not ready|block|status)'
 
   - name: plan-validator-coverage-matrix
     prompt: |

scripts/evals/Build-AgentBehaviorSpec.ps1

@@ -351,6 +351,8 @@ function Invoke-AgentBehaviorSpecCore {
         $diffText = Get-LineDiff -Expected $rendered -Actual $existingText -Path $OutputPath
         [System.IO.File]::WriteAllText($DriftDiffPath, $diffText)
         Write-Host "drift detected; diff written to $DriftDiffPath" -ForegroundColor Yellow
+        Write-Host "To fix: regenerate the spec from the stimuli partials and commit it:" -ForegroundColor Yellow
+        Write-Host "  pwsh -NoProfile -File scripts/evals/Build-AgentBehaviorSpec.ps1" -ForegroundColor Yellow
         return [pscustomobject]@{ Outcome = 'Drift'; OutputPath = $OutputPath; DiffPath = $DriftDiffPath }
     }