feat(action): refactor github action input surface around args; harden env passthrough

.github/workflows/pr-checks.yml

@@ -997,22 +997,37 @@ jobs:
         id: dogfood
         uses: ./
         with:
+          # Exercises what tests/test_action_inputs.py cannot: it asserts the argv
+          # both step scripts build, but only a real runner proves that
+          # `working-directory:` works on a composite step and that a plugin
+          # installed via `--with` is discovered at runtime. Hence a relative
+          # run-dir (landing under tasks/), the bare task filename and the `../`
+          # plugin path, all resolved from `working-directory`.
           version: local
-          tasks: tasks/hello_date.yaml
-          model: claude-haiku-4-5-20251001
+          working-directory: tasks
           run-dir: runs/ci-action-dogfood
-          junit-path: runs/ci-action-dogfood/junit.xml
-          # Credentials go through the generic env passthrough (the only channel);
+          extra-packages: ../tests/fixtures/byoa_demo_plugin
+          # The bracketed `-D` override is the case one-argv-entry-per-line exists
+          # for: `[...]` is a bash character class, so a whitespace-split input
+          # would collapse the list whenever a file in the cwd matched. It adds
+          # Glob to hello_date.yaml's three tools, so the assertions below can tell
+          # "arrived" from "ignored".
+          args: |
+            hello_date.yaml
+            --model
+            claude-haiku-4-5-20251001
+            -D
+            agent.allowed_tools=[Read,Write,Bash,Glob]
           # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task
-          # succeeding. A floor of 0.0 passes for any produced score (exercises
-          # the gate path green in CI without flakiness); the second line
-          # exercises multi-line env parsing.
-          minimum-task-score: "0.0"
+          # succeeding; the second line exercises multi-line env parsing.
           env: |
             ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
             CE_DOGFOOD_MARKER=1
 
+      # In `tasks/` because that is the assertion: `run-dir` is reported exactly as
+      # passed, so a relative one is relative to `working-directory`.
       - name: Verify outputs and JUnit file
+        working-directory: tasks
         env:
           JUNIT: ${{ steps.dogfood.outputs.junit-path }}
           RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
@@ -1023,11 +1038,66 @@ jobs:
           # our writer emits no DTDs/entities) — stdlib ET is fine here.
           python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT"
           test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; }
+          # An absolute path would mean working-directory was ignored.
+          case "$RUNDIR" in /*) echo "run-dir output was rewritten to an absolute path: $RUNDIR"; exit 1 ;; esac
+
+      # The action does not touch $GITHUB_STEP_SUMMARY: this is both the assertion
+      # that `run-md-path` is real and the recipe the docs hand consumers.
+      - name: Append the run report to the job summary
+        if: always()
+        working-directory: tasks
+        env:
+          RUN_MD: ${{ steps.dogfood.outputs.run-md-path }}
+        run: |
+          set -euo pipefail
+          test -f "$RUN_MD" || { echo "run-md-path output does not exist: $RUN_MD"; exit 1; }
+          cat "$RUN_MD" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Verify the plugin was discovered and the bracketed override arrived
+        working-directory: tasks
+        env:
+          RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
+        run: |
+          set -euo pipefail
+
+          # `coder-eval` on PATH is the action's uv tool shim, so this interrogates
+          # its environment: a task naming the fixture's agent kind validates only
+          # if the entry point was discovered there, else plan exits 1 with
+          # "No agent registered for type 'byoa-demo'".
+          cat > byoa-probe.yaml <<'YAML'
+          task_id: "action_extra_packages_probe"
+          description: "Validates only when the byoa-demo plugin is discoverable."
+          initial_prompt: "not executed - plan validates without running an agent"
+          agent:
+            type: "byoa-demo"
+          success_criteria:
+            - type: "file_exists"
+              path: "app.py"
+              description: "not executed"
+          YAML
+          coder-eval plan byoa-probe.yaml
+          rm -f byoa-probe.yaml
+
+          # And the `-D` value survived as a 4-element list, not word-split or
+          # glob-rewritten.
+          RUN_JSON="$RUNDIR/run.json" python3 <<'PY'
+          import json, os, sys
+
+          data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8"))
+          rows = data.get("task_results") or []
+          if not rows:
+              sys.exit("run.json has no task_results to check the -D override against")
+          tools = (rows[0].get("agent_config") or {}).get("allowed_tools")
+          expected = ["Read", "Write", "Bash", "Glob"]
+          if tools != expected:
+              sys.exit(f"-D override did not arrive intact: allowed_tools={tools!r}, expected {expected!r}")
+          print(f"bracketed -D override resolved to {tools!r}")
+          PY
 
       - name: Upload dogfood run on failure
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: action-dogfood-runs
-          path: runs/ci-action-dogfood/
+          path: tasks/runs/ci-action-dogfood/
           retention-days: 7

.github/workflows/verify-published-action.yml

@@ -359,25 +359,24 @@ jobs:
           echo "--- task YAML:"; cat tasks/published_smoke.yaml
 
       # continue-on-error, because this step's exit code is NOT the gate. The action
-      # exits with coder-eval's own code (action.yml combines them), and coder-eval
-      # exits 1 on any failed task -- so a model flake failing `file_exists` would
-      # redden this workflow even with minimum-task-score at 0.0, which does not
-      # neutralize that path. This check must answer "does the published action still
-      # work", not "is the model still good": the verification step below gates on
-      # ARTIFACTS instead. A genuine model/credential outage still surfaces there, via
-      # the zero-token assertion.
+      # exits with coder-eval's own code, and coder-eval exits 1 on any failed task,
+      # so a model flake failing `file_exists` would redden this workflow. This check
+      # must answer "does the published action still work", not "is the model still
+      # good": the verification step below gates on ARTIFACTS instead. A genuine
+      # model/credential outage still surfaces there, via the zero-token assertion.
       - name: Run the published action
         id: run
         continue-on-error: true
         uses: UiPath/coder_eval@v0   # major asserted by the preflight job above
         with:
           # `version:` intentionally omitted -- the whole point is to exercise the
           # default pin baked into action.yml at the v0 tag.
-          tasks: tasks/published_smoke.yaml
-          model: claude-haiku-4-5-20251001
           run-dir: runs/verify-published
-          junit-path: runs/verify-published/junit.xml
-          minimum-task-score: "0.0"
+          # Task path and flags both go through `args` — the action promotes no CLI flag.
+          args: |
+            tasks/published_smoke.yaml
+            --model
+            claude-haiku-4-5-20251001
           env: |
             ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
 

README.md

@@ -117,44 +117,54 @@ That adds six slash commands: `/coder-eval:init`, `/coder-eval:check-skill`,
 A composite action — on the Marketplace as
 [**coder_eval**](https://github.com/marketplace/actions/coder_eval) — runs
 `coder-eval` as a CI gate. It installs the pinned CLI, runs your tasks, writes a
-JUnit XML report, appends `run.md` to the job summary, and fails the step on any
-task/gate failure:
+JUnit XML report, reports where its artifacts landed, and fails the step on any
+task failure:
 
 ```yaml
 - uses: actions/setup-node@v4      # the claude-code agent needs the Claude CLI…
   with: { node-version: '20' }
 - run: npm install -g @anthropic-ai/claude-code
 
 - uses: UiPath/coder_eval@v0       # …then run the gate (@v1 once 1.0.0 ships; @vX.Y.Z pins exactly)
+  id: eval
   with:
-    tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml
-    model: claude-sonnet-5
+    args: |
+      tests/tasks/**/*.yaml
+      --model
+      claude-sonnet-5
     env: |
       ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
 ```
 
+Eight inputs, and **none of them is a `coder-eval run` flag**. The CLI has 21;
+GitHub silently ignores an input the referenced tag does not define, so a
+forwarding input that is mistyped or newer than your pin yields a run that
+measured something else and still exits 0. A wrong CLI flag is a hard error. So
+flags and task globs all go through `args`, and an input exists only where the
+action does something with the value besides pass it along.
+
 | Input | Default | Purpose |
 | --- | --- | --- |
-| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob |
-| `tags` | — | `--tags` filter |
-| `model` | — | `--model` override |
-| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) |
+| `args` | — | Task paths/globs and every flag for `coder-eval run`, one argument per line, verbatim |
 | `version` | pinned release | PyPI version, or `local` to install from the checkout |
-| `run-dir` | `runs/ci` | Run directory |
-| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report |
-| `step-summary` | `true` | Append `run.md` to the job summary |
+| `extras` | — | coder-eval extras, composed into the install requirement (`codex`, `antigravity,litellm`) |
+| `extra-packages` | — | Extra requirements installed into coder-eval's environment (`--with`), one per line |
+| `install-flags` | — | Flags for `uv tool install`, one per line (`--prerelease=allow`, `--extra-index-url …`) |
 | `env` | — | Credentials/backend passthrough: newline-separated `NAME=VALUE` pairs, exported for the run step only |
-| `minimum-task-score` | *(off)* | Strict floor (0.0–1.0): fail the step if any task's `weighted_score` is below it |
+| `working-directory` | `.` | Directory every step of the action runs in |
+| `run-dir` | `runs/ci` | Run directory; also where the reports are written |
 
-Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's
-test-report renderer — e.g. on GitHub Actions with
-[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report):
+Outputs: `run-dir`, `junit-path` (`<run-dir>/junit.xml`) and `run-md-path`
+(`<run-dir>/run.md`). The action writes nothing to the job summary — a consumer
+that has to redact the report first cannot undo a write that already happened:
 
 ```yaml
+- if: always()
+  run: cat "${{ steps.eval.outputs.run-md-path }}" >> "$GITHUB_STEP_SUMMARY"
 - uses: mikepenz/action-junit-report@v5
   if: always()
   with:
-    report_paths: coder-eval-junit.xml
+    report_paths: ${{ steps.eval.outputs.junit-path }}
 ```
 
 **Credentials and backend config** are the sole responsibility of `env` — a
@@ -164,17 +174,13 @@ it can't leak into later steps). Set whatever the run needs, Anthropic or not:
 ```yaml
 - uses: UiPath/coder_eval@v0
   with:
-    tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml
-    minimum-task-score: "0.8"   # fail the build if any task scores below 0.8
+    args: tests/tasks/**/*.yaml
     env: |
       API_BACKEND=bedrock
       AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }}
 ```
 
-`minimum-task-score` is a strict floor **on top of** coder-eval's own exit
-code: the step fails if *either* coder-eval exits non-zero *or* any task's
-`weighted_score` falls below the floor. Leave it unset to gate on the exit code
-alone.
+The step's exit code is coder-eval's own: non-zero on any failed task.
 
 > **Agent runtime is the caller's responsibility.** The action is agent-agnostic —
 > it installs `coder-eval` but no coding-agent runtime, which is why the example
@@ -215,7 +221,7 @@ alone.
 | [Bring Your Own Dataset](docs/DATASETS.md) | Fan a single task out over a dataset |
 | [Dialog Mode](docs/DIALOG_MODE.md) | Evaluate agents in multi-turn conversation via a simulated user |
 | [Docker Isolation](docs/DOCKER_ISOLATION.md) | The container sandbox driver, with custom images |
-| [CI Gate & GitHub Action](docs/CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, score floor |
+| [CI Gate & GitHub Action](docs/CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports |
 | [Claude Code Plugin](docs/PLUGIN.md) | Install the Claude Code plugin — author, run, and analyze suites from inside the agent |
 | [Extending Coder Eval](docs/EXTENDING.md) | Author a custom agent, criterion, or model pricing via the plugin SPI |
 | [Report Schema](docs/REPORT_SCHEMA.md) | Field-level reference for run.json / variant.json / task.json |