diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 94cce6d3..3bf52e59 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -997,22 +997,37 @@ jobs: id: dogfood uses: ./ with: + # Exercises what tests/test_action_inputs.py cannot: it asserts the argv + # both step scripts build, but only a real runner proves that + # `working-directory:` works on a composite step and that a plugin + # installed via `--with` is discovered at runtime. Hence a relative + # run-dir (landing under tasks/), the bare task filename and the `../` + # plugin path, all resolved from `working-directory`. version: local - tasks: tasks/hello_date.yaml - model: claude-haiku-4-5-20251001 + working-directory: tasks run-dir: runs/ci-action-dogfood - junit-path: runs/ci-action-dogfood/junit.xml - # Credentials go through the generic env passthrough (the only channel); + extra-packages: ../tests/fixtures/byoa_demo_plugin + # The bracketed `-D` override is the case one-argv-entry-per-line exists + # for: `[...]` is a bash character class, so a whitespace-split input + # would collapse the list whenever a file in the cwd matched. It adds + # Glob to hello_date.yaml's three tools, so the assertions below can tell + # "arrived" from "ignored". + args: | + hello_date.yaml + --model + claude-haiku-4-5-20251001 + -D + agent.allowed_tools=[Read,Write,Bash,Glob] # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task - # succeeding. A floor of 0.0 passes for any produced score (exercises - # the gate path green in CI without flakiness); the second line - # exercises multi-line env parsing. - minimum-task-score: "0.0" + # succeeding; the second line exercises multi-line env parsing. env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} CE_DOGFOOD_MARKER=1 + # In `tasks/` because that is the assertion: `run-dir` is reported exactly as + # passed, so a relative one is relative to `working-directory`. - name: Verify outputs and JUnit file + working-directory: tasks env: JUNIT: ${{ steps.dogfood.outputs.junit-path }} RUNDIR: ${{ steps.dogfood.outputs.run-dir }} @@ -1023,11 +1038,66 @@ jobs: # our writer emits no DTDs/entities) — stdlib ET is fine here. python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; } + # An absolute path would mean working-directory was ignored. + case "$RUNDIR" in /*) echo "run-dir output was rewritten to an absolute path: $RUNDIR"; exit 1 ;; esac + + # The action does not touch $GITHUB_STEP_SUMMARY: this is both the assertion + # that `run-md-path` is real and the recipe the docs hand consumers. + - name: Append the run report to the job summary + if: always() + working-directory: tasks + env: + RUN_MD: ${{ steps.dogfood.outputs.run-md-path }} + run: | + set -euo pipefail + test -f "$RUN_MD" || { echo "run-md-path output does not exist: $RUN_MD"; exit 1; } + cat "$RUN_MD" >> "$GITHUB_STEP_SUMMARY" + + - name: Verify the plugin was discovered and the bracketed override arrived + working-directory: tasks + env: + RUNDIR: ${{ steps.dogfood.outputs.run-dir }} + run: | + set -euo pipefail + + # `coder-eval` on PATH is the action's uv tool shim, so this interrogates + # its environment: a task naming the fixture's agent kind validates only + # if the entry point was discovered there, else plan exits 1 with + # "No agent registered for type 'byoa-demo'". + cat > byoa-probe.yaml <<'YAML' + task_id: "action_extra_packages_probe" + description: "Validates only when the byoa-demo plugin is discoverable." + initial_prompt: "not executed - plan validates without running an agent" + agent: + type: "byoa-demo" + success_criteria: + - type: "file_exists" + path: "app.py" + description: "not executed" + YAML + coder-eval plan byoa-probe.yaml + rm -f byoa-probe.yaml + + # And the `-D` value survived as a 4-element list, not word-split or + # glob-rewritten. + RUN_JSON="$RUNDIR/run.json" python3 <<'PY' + import json, os, sys + + data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) + rows = data.get("task_results") or [] + if not rows: + sys.exit("run.json has no task_results to check the -D override against") + tools = (rows[0].get("agent_config") or {}).get("allowed_tools") + expected = ["Read", "Write", "Bash", "Glob"] + if tools != expected: + sys.exit(f"-D override did not arrive intact: allowed_tools={tools!r}, expected {expected!r}") + print(f"bracketed -D override resolved to {tools!r}") + PY - name: Upload dogfood run on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: action-dogfood-runs - path: runs/ci-action-dogfood/ + path: tasks/runs/ci-action-dogfood/ retention-days: 7 diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index d625fa40..134dbeb1 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -359,13 +359,11 @@ jobs: echo "--- task YAML:"; cat tasks/published_smoke.yaml # continue-on-error, because this step's exit code is NOT the gate. The action - # exits with coder-eval's own code (action.yml combines them), and coder-eval - # exits 1 on any failed task -- so a model flake failing `file_exists` would - # redden this workflow even with minimum-task-score at 0.0, which does not - # neutralize that path. This check must answer "does the published action still - # work", not "is the model still good": the verification step below gates on - # ARTIFACTS instead. A genuine model/credential outage still surfaces there, via - # the zero-token assertion. + # exits with coder-eval's own code, and coder-eval exits 1 on any failed task, + # so a model flake failing `file_exists` would redden this workflow. This check + # must answer "does the published action still work", not "is the model still + # good": the verification step below gates on ARTIFACTS instead. A genuine + # model/credential outage still surfaces there, via the zero-token assertion. - name: Run the published action id: run continue-on-error: true @@ -373,11 +371,12 @@ jobs: with: # `version:` intentionally omitted -- the whole point is to exercise the # default pin baked into action.yml at the v0 tag. - tasks: tasks/published_smoke.yaml - model: claude-haiku-4-5-20251001 run-dir: runs/verify-published - junit-path: runs/verify-published/junit.xml - minimum-task-score: "0.0" + # Task path and flags both go through `args` — the action promotes no CLI flag. + args: | + tasks/published_smoke.yaml + --model + claude-haiku-4-5-20251001 env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} diff --git a/README.md b/README.md index fe7adb51..1273b57c 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,8 @@ That adds six slash commands: `/coder-eval:init`, `/coder-eval:check-skill`, A composite action — on the Marketplace as [**coder_eval**](https://github.com/marketplace/actions/coder_eval) — runs `coder-eval` as a CI gate. It installs the pinned CLI, runs your tasks, writes a -JUnit XML report, appends `run.md` to the job summary, and fails the step on any -task/gate failure: +JUnit XML report, reports where its artifacts landed, and fails the step on any +task failure: ```yaml - uses: actions/setup-node@v4 # the claude-code agent needs the Claude CLI… @@ -126,35 +126,45 @@ task/gate failure: - run: npm install -g @anthropic-ai/claude-code - uses: UiPath/coder_eval@v0 # …then run the gate (@v1 once 1.0.0 ships; @vX.Y.Z pins exactly) + id: eval with: - tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml - model: claude-sonnet-5 + args: | + tests/tasks/**/*.yaml + --model + claude-sonnet-5 env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` +Eight inputs, and **none of them is a `coder-eval run` flag**. The CLI has 21; +GitHub silently ignores an input the referenced tag does not define, so a +forwarding input that is mistyped or newer than your pin yields a run that +measured something else and still exits 0. A wrong CLI flag is a hard error. So +flags and task globs all go through `args`, and an input exists only where the +action does something with the value besides pass it along. + | Input | Default | Purpose | | --- | --- | --- | -| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob | -| `tags` | — | `--tags` filter | -| `model` | — | `--model` override | -| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) | +| `args` | — | Task paths/globs and every flag for `coder-eval run`, one argument per line, verbatim | | `version` | pinned release | PyPI version, or `local` to install from the checkout | -| `run-dir` | `runs/ci` | Run directory | -| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | -| `step-summary` | `true` | Append `run.md` to the job summary | +| `extras` | — | coder-eval extras, composed into the install requirement (`codex`, `antigravity,litellm`) | +| `extra-packages` | — | Extra requirements installed into coder-eval's environment (`--with`), one per line | +| `install-flags` | — | Flags for `uv tool install`, one per line (`--prerelease=allow`, `--extra-index-url …`) | | `env` | — | Credentials/backend passthrough: newline-separated `NAME=VALUE` pairs, exported for the run step only | -| `minimum-task-score` | *(off)* | Strict floor (0.0–1.0): fail the step if any task's `weighted_score` is below it | +| `working-directory` | `.` | Directory every step of the action runs in | +| `run-dir` | `runs/ci` | Run directory; also where the reports are written | -Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's -test-report renderer — e.g. on GitHub Actions with -[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): +Outputs: `run-dir`, `junit-path` (`/junit.xml`) and `run-md-path` +(`/run.md`). The action writes nothing to the job summary — a consumer +that has to redact the report first cannot undo a write that already happened: ```yaml +- if: always() + run: cat "${{ steps.eval.outputs.run-md-path }}" >> "$GITHUB_STEP_SUMMARY" - uses: mikepenz/action-junit-report@v5 if: always() with: - report_paths: coder-eval-junit.xml + report_paths: ${{ steps.eval.outputs.junit-path }} ``` **Credentials and backend config** are the sole responsibility of `env` — a @@ -164,17 +174,13 @@ it can't leak into later steps). Set whatever the run needs, Anthropic or not: ```yaml - uses: UiPath/coder_eval@v0 with: - tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml - minimum-task-score: "0.8" # fail the build if any task scores below 0.8 + args: tests/tasks/**/*.yaml env: | API_BACKEND=bedrock AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }} ``` -`minimum-task-score` is a strict floor **on top of** coder-eval's own exit -code: the step fails if *either* coder-eval exits non-zero *or* any task's -`weighted_score` falls below the floor. Leave it unset to gate on the exit code -alone. +The step's exit code is coder-eval's own: non-zero on any failed task. > **Agent runtime is the caller's responsibility.** The action is agent-agnostic — > it installs `coder-eval` but no coding-agent runtime, which is why the example @@ -215,7 +221,7 @@ alone. | [Bring Your Own Dataset](docs/DATASETS.md) | Fan a single task out over a dataset | | [Dialog Mode](docs/DIALOG_MODE.md) | Evaluate agents in multi-turn conversation via a simulated user | | [Docker Isolation](docs/DOCKER_ISOLATION.md) | The container sandbox driver, with custom images | -| [CI Gate & GitHub Action](docs/CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, score floor | +| [CI Gate & GitHub Action](docs/CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports | | [Claude Code Plugin](docs/PLUGIN.md) | Install the Claude Code plugin — author, run, and analyze suites from inside the agent | | [Extending Coder Eval](docs/EXTENDING.md) | Author a custom agent, criterion, or model pricing via the plugin SPI | | [Report Schema](docs/REPORT_SCHEMA.md) | Field-level reference for run.json / variant.json / task.json | diff --git a/action.yml b/action.yml index 9747745f..81e8417a 100644 --- a/action.yml +++ b/action.yml @@ -1,88 +1,111 @@ -# `name` is the GitHub Marketplace listing title and must be globally unique -# across Marketplace actions, users, AND organizations. `coder-eval` is taken by -# an unrelated squatted org (github.com/coder-eval), so the listing uses the -# underscored repo name instead. This value is display-only: consumers reference -# the action by repo path (`uses: UiPath/coder_eval@v0`), never by this name. +# Marketplace listing titles are globally unique across actions, users AND orgs, +# and `coder-eval` is taken by an unrelated org, hence the underscored repo name. +# Display only: consumers reference the action by repo path. name: coder_eval -# Matches the authorship the project already declares in pyproject.toml -# (`authors = [{ name = "UiPath", ... }]`) and NOTICE (`© 2026 UiPath`). author: UiPath -description: Run coder-eval evaluation tasks as a CI gate, with JUnit XML output and a job-summary report. +description: Install a pinned coder-eval and run evaluation tasks as a CI gate, with JUnit XML output. branding: icon: check-circle color: orange -# This action installs and runs the `coder-eval` CLI. It is agent-agnostic: it -# does NOT install any coding-agent runtime. Tasks that use the default -# `claude-code` agent need the `claude` CLI on PATH (Node + the -# `@anthropic-ai/claude-code` npm package) provided by the calling job before -# this action runs. See the README "Use as a GitHub Action" section. +# Agent-agnostic: this installs `coder-eval`, not any coding-agent runtime. Tasks +# on the default `claude-code` agent need the `claude` CLI on PATH, supplied by +# the calling job. See the README's "Use as a GitHub Action". # # SECURITY: evaluated tasks execute agent-generated code. Do NOT run this action # under `pull_request_target` with secrets exposed to untrusted fork PRs. - +# +# No input forwards a `coder-eval run` flag, deliberately: GitHub silently +# IGNORES an input the referenced tag does not define, so a forwarding input that +# is mistyped or newer than the pinned tag yields a green run that measured +# something else, where a wrong CLI flag is a hard error. Flags go through `args`. +# An input lives here only when the action uses the value for more than passing +# it on. inputs: - tasks: - description: Task YAML path(s)/glob passed to `coder-eval run` (empty = all tasks/ recursively) + version: + description: coder-eval version to install from PyPI, or "local" to install from the action checkout required: false - default: "" - tags: - description: Only run tasks matching any of these comma-separated tags (--tags) + default: "0.11.5" # <-- kept in sync with releases by release.yml + extras: + description: >- + Comma-separated coder-eval extras (`codex`, `antigravity,litellm`), composed + into the install requirement rather than installed afterwards: `uv tool + install` builds an isolated environment whose shims shadow anything else + named `coder-eval` on PATH. Each name must match + ^[A-Za-z0-9][A-Za-z0-9._-]*$. required: false default: "" - model: - description: Override agent model for all tasks (--model) + extra-packages: + description: >- + Extra requirements installed INTO coder-eval's tool environment (`uv tool + install --with`), one per line: a PEP 508 specifier, or a path relative to + `working-directory`. The only way an out-of-tree coder-eval plugin becomes + discoverable, since an entry point is found only when the plugin shares a + virtualenv with its host. required: false default: "" - extra-args: - description: Extra arguments appended verbatim to `coder-eval run` (trusted caller input; covers --experiment, -D overrides, --tags exclusions, etc.) + install-flags: + description: >- + Flags for `uv tool install`, one per line, appended verbatim — resolver + control this action does not model (`--prerelease=allow`, + `--extra-index-url `). required: false default: "" - version: - description: coder-eval version to install from PyPI, or "local" to install from the action checkout - required: false - default: "0.11.5" # <-- kept in sync with releases by release.yml - run-dir: - description: Run directory (--run-dir) - required: false - default: "runs/ci" - junit-path: - description: Where to write the JUnit XML report - required: false - default: "coder-eval-junit.xml" - step-summary: - description: Append run.md to the GitHub job summary ("true"/"false") + args: + description: >- + Arguments appended to `coder-eval run`, ONE PER LINE, each verbatim — no + word splitting, no pathname expansion. The only channel for the CLI's flags + and for the task paths themselves. A flag and its value are two lines (`-D`, + then `key=[a,b]`) or one line in `=` form (`--model=x`); sharing a line + makes one malformed token. Globs reach the CLI unexpanded and it expands + them itself (`**` included), exiting 1 if nothing matches. Blank lines, `#` + comments and surrounding whitespace are ignored here, in `install-flags` and + in `extra-packages`. required: false - default: "true" + default: "" env: description: >- - Environment passthrough — newline-separated NAME=VALUE pairs exported for - the coder-eval process only (scoped to the run step; NOT written to - $GITHUB_ENV, so nothing leaks into later job steps). This is the sole - channel for credentials and backend config: set ANTHROPIC_API_KEY, - API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names - must match ^[A-Za-z_][A-Za-z0-9_]*$; blank lines and `#` comments are - ignored. Wire values from repository secrets (secrets.MY_KEY) — never - inline a secret literal. + Newline-separated NAME=VALUE pairs, handed to the coder-eval process only. + Never written to $GITHUB_ENV and never exported into the action's own + shell, so a forwarded secret cannot bleed into later job steps, nor rewrite + what this step runs. The sole channel for credentials and backend config + (ANTHROPIC_API_KEY, API_BACKEND, model vars, plugin paths). Names must match + ^[A-Za-z_][A-Za-z0-9_]*$; PATH and the loader/shell-startup names are + rejected outright (use $GITHUB_PATH for a tool directory). Wire values from + repository secrets; never inline a literal. required: false default: "" - minimum-task-score: + working-directory: description: >- - Optional strict floor (0.0–1.0): EVERY scored task, in every variant, must - reach it or the step fails. A gate ON TOP OF coder-eval's own exit code — - the step fails if EITHER coder-eval exits non-zero OR any task's - weighted_score is below this. Empty (the default) disables the floor, - leaving coder-eval's exit code the sole gate. + Directory every step of this action runs in, and what `run-dir`, the task + paths in `args` and relative `extra-packages` entries resolve against. + GitHub rejects `working-directory:` on a `uses:` step and a job-level + `defaults.run` does not reach inside a composite, so this input is the only + way to run the gate from a subdirectory. required: false - default: "" + default: "." + run-dir: + description: >- + Run directory (--run-dir), reported as given. The JUnit and markdown reports + are written inside it and reported as outputs, so there is no separate + `junit-path` input. + required: false + default: "runs/ci" outputs: run-dir: - description: The run directory containing run.json/run.md + description: The run directory, as given, containing run.json/run.md value: ${{ steps.run.outputs.run-dir }} junit-path: - description: Path to the written JUnit XML report + description: Path to the written JUnit XML report (`/junit.xml`) value: ${{ steps.run.outputs.junit-path }} + run-md-path: + description: >- + Path to the markdown run report (`/run.md`). The action does not + append it to $GITHUB_STEP_SUMMARY, because a consumer that has to redact the + report first cannot undo a write that already happened; + `cat "$RUN_MD" >> "$GITHUB_STEP_SUMMARY"` is the whole of what that replaces. + value: ${{ steps.run.outputs.run-md-path }} runs: using: composite @@ -91,37 +114,101 @@ runs: uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - name: Install coder-eval shell: bash + working-directory: ${{ inputs.working-directory }} env: CE_VERSION: ${{ inputs.version }} + CE_EXTRAS: ${{ inputs.extras }} + CE_EXTRA_PACKAGES: ${{ inputs.extra-packages }} + CE_INSTALL_FLAGS: ${{ inputs.install-flags }} CE_ACTION_PATH: ${{ github.action_path }} run: | set -euo pipefail + + # The one line-list parser for `install-flags`, `extra-packages` and `args`. + # Emits to stdout instead of filling a nameref array, which bash 3.2 (macOS + # runners) lacks. tests/test_action_inputs.py asserts the two copies of this + # function are byte-identical. + clean_lines() { + local line + while IFS= read -r line; do + line="${line%$'\r'}" # tolerate CRLF inputs + line="${line#"${line%%[![:space:]]*}"}" # left-trim + line="${line%"${line##*[![:space:]]}"}" # right-trim + [ -z "$line" ] && continue + case "$line" in '#'*) continue ;; esac # allow comment lines + printf '%s\n' "$line" + done + } + + # Into the requirement string rather than a follow-up install: the tool + # environment's shims shadow any other coder-eval on PATH, so an extra + # added beside it is never imported. Validated, since it reaches a resolver. + extras="" + if [ -n "$CE_EXTRAS" ]; then + if [[ ! "$CE_EXTRAS" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*(,[A-Za-z0-9][A-Za-z0-9._-]*)*$ ]]; then + echo "::error::extras must be a comma-separated list of extra names, got '$CE_EXTRAS'"; exit 1 + fi + extras="[$CE_EXTRAS]" + fi + + install_args=(tool install) + while IFS= read -r flag; do + install_args+=("$flag") + done < <(clean_lines <<< "$CE_INSTALL_FLAGS") + + # One argv entry each: a path can contain spaces and a specifier `[`, `]` + # or `>`, none of which survive word splitting. + while IFS= read -r req; do + install_args+=(--with "$req") + done < <(clean_lines <<< "$CE_EXTRA_PACKAGES") + if [ "$CE_VERSION" = "local" ]; then - uv tool install "$CE_ACTION_PATH" + install_args+=("${CE_ACTION_PATH}${extras}") else - uv tool install "coder-eval==$CE_VERSION" + install_args+=("coder-eval${extras}==${CE_VERSION}") fi + uv "${install_args[@]}" - name: Run coder-eval id: run shell: bash + working-directory: ${{ inputs.working-directory }} env: - CE_TASKS: ${{ inputs.tasks }} - CE_TAGS: ${{ inputs.tags }} - CE_MODEL: ${{ inputs.model }} - CE_EXTRA_ARGS: ${{ inputs.extra-args }} + CE_ARGS: ${{ inputs.args }} CE_RUN_DIR: ${{ inputs.run-dir }} - CE_JUNIT: ${{ inputs.junit-path }} - CE_SUMMARY: ${{ inputs.step-summary }} CE_ENV: ${{ inputs.env }} - CE_MIN_SCORE: ${{ inputs.minimum-task-score }} run: | set -uo pipefail - # Generic env passthrough. Each NAME=VALUE line is exported for the - # coder-eval child of THIS step only — deliberately not written to - # $GITHUB_ENV, so a forwarded secret never bleeds into later job steps. - # Only NAME is validated; VALUE is treated as opaque data (never eval'd), - # so no crafted value can inject shell. + # The one line-list parser for `install-flags`, `extra-packages` and `args`. + # Emits to stdout instead of filling a nameref array, which bash 3.2 (macOS + # runners) lacks. tests/test_action_inputs.py asserts the two copies of this + # function are byte-identical. + clean_lines() { + local line + while IFS= read -r line; do + line="${line%$'\r'}" # tolerate CRLF inputs + line="${line#"${line%%[![:space:]]*}"}" # left-trim + line="${line%"${line##*[![:space:]]}"}" # right-trim + [ -z "$line" ] && continue + case "$line" in '#'*) continue ;; esac # allow comment lines + printf '%s\n' "$line" + done + } + + # COLLECTED, not exported. `export` mutates THIS shell, and the name + # filter below admits `CE_ARGS`, `CE_RUN_DIR` and `PATH`, all of which + # are read after this loop. The reachable case is not a hostile workflow author + # but a VALUE carrying a newline: the loop is line-based, so one + # forwarded secret or interpolated workflow expression whose value holds + # `\nCE_RUN_DIR=...` became a second honoured entry that could rewrite + # the argv, redirect where results land, or shadow which coder-eval ran. + # Handing the pairs to `env` instead makes the documented contract + # ("for the coder-eval process only") true rather than aspirational. + # + # Only NAME is validated; VALUE is opaque data and is never eval'd. Not + # clean_lines, because right-trimming would silently alter a credential + # that legitimately ends in whitespace. + ce_env=() n=0 while IFS= read -r line; do n=$((n + 1)) @@ -129,9 +216,8 @@ runs: line="${line#"${line%%[![:space:]]*}"}" # left-trim [ -z "$line" ] && continue case "$line" in '#'*) continue ;; esac # allow comment lines - # Never echo $line/$value on error — a caller who forgets the `NAME=` - # prefix would otherwise print a forwarded secret verbatim (GitHub - # only masks values it knows are secrets). Report by position instead. + # Report by position, never echoing the line: a caller who forgot the + # `NAME=` prefix would otherwise print a secret GitHub cannot mask. if [ "$line" = "${line#*=}" ]; then echo "::error::env entry #$n is not NAME=VALUE (no '=' found)"; exit 1 fi @@ -139,79 +225,44 @@ runs: if [[ ! "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then echo "::error::env entry #$n has an invalid name (must match ^[A-Za-z_][A-Za-z0-9_]*\$)"; exit 1 fi - export "$name=${line#*=}" + # Rejected by name: these change how the child RESOLVES code, not how + # it behaves, and this channel exists for credentials and backend + # config. A tool directory belongs on $GITHUB_PATH in a step of your + # own, which is scoped and visible in the log. + case "$name" in + PATH|IFS|ENV|BASH_ENV|SHELLOPTS|BASHOPTS|LD_PRELOAD|LD_LIBRARY_PATH|DYLD_INSERT_LIBRARIES|DYLD_LIBRARY_PATH) + echo "::error::env entry #$n uses the reserved name '$name'. Put a tool directory on \$GITHUB_PATH in an earlier step instead."; exit 1 ;; + esac + ce_env+=("$name=${line#*=}") done <<< "$CE_ENV" - args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") - [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") - [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") - # extra-args is a trusted caller input, split on whitespace intentionally - # shellcheck disable=SC2206 - [ -n "$CE_EXTRA_ARGS" ] && args+=($CE_EXTRA_ARGS) - # shellcheck disable=SC2206 - [ -n "$CE_TASKS" ] && args+=($CE_TASKS) + # Reports live inside the run directory; `%/` keeps a trailing slash on the + # input from doubling the separator in the reported paths. + run_dir="${CE_RUN_DIR%/}" + junit="$run_dir/junit.xml" + run_md="$run_dir/run.md" + + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$junit") + while IFS= read -r arg; do + args+=("$arg") + done < <(clean_lines <<< "$CE_ARGS") + set +e - coder-eval "${args[@]}" + # `${a[@]+"${a[@]}"}` because bash 3.2 (macOS runners) treats an empty + # array as unset under `set -u`. `--` so a NAME=VALUE pair can never be + # read as an option to env itself. + env -- ${ce_env[@]+"${ce_env[@]}"} coder-eval "${args[@]}" CODE=$? set -e - echo "run-dir=$CE_RUN_DIR" >> "$GITHUB_OUTPUT" - echo "junit-path=$CE_JUNIT" >> "$GITHUB_OUTPUT" - if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then - head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" - fi - # Optional per-task score floor. Reads the always-written run.json spine - # (task_results[*].weighted_score) rather than experiment.json, so it - # works for plain and experiment runs alike. A gate ON TOP OF CODE: - # empty CE_MIN_SCORE disables it, leaving CODE the sole verdict. Runs - # regardless of CODE (run.json is emitted even on a red run), then the - # two verdicts are combined so both surface. - GATE=0 - if [ -n "$CE_MIN_SCORE" ]; then - if [ ! -f "$CE_RUN_DIR/run.json" ]; then - echo "::error::score gate: $CE_RUN_DIR/run.json not found (cannot verify minimum-task-score=$CE_MIN_SCORE)" - GATE=1 - else - RUN_JSON="$CE_RUN_DIR/run.json" MIN_SCORE="$CE_MIN_SCORE" python3 <<'PY' || GATE=1 - import json, math, os, sys + # Written before the exit so a red run still reports where its artifacts + # are. Output propagation from a failed composite step is not a documented + # guarantee, so a consumer chasing a failed run should key off the paths it + # passed in rather than these. + { + echo "run-dir=$CE_RUN_DIR" + echo "junit-path=$junit" + echo "run-md-path=$run_md" + } >> "$GITHUB_OUTPUT" - raw = os.environ["MIN_SCORE"] - try: - floor = float(raw) - except ValueError: - print(f"::error::minimum-task-score must be a number, got '{raw}'") - sys.exit(1) - if not math.isfinite(floor) or not (0.0 <= floor <= 1.0): - print(f"::error::minimum-task-score must be within [0.0, 1.0], got '{raw}'") - sys.exit(1) - data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) - rows = [] - for r in data.get("task_results", []): - s = r.get("weighted_score") - # bool is an int subclass — exclude it; skip errored rows (score None), - # which coder-eval's own exit code already accounts for; and skip - # non-finite (NaN/inf) — json.loads accepts them, and NaN makes - # min()/>= order-dependent, which could silently MASK a below-floor - # task. A blob-pulled/malformed run.json must fail closed, not pass. - if isinstance(s, (int, float)) and not isinstance(s, bool) and math.isfinite(s): - rows.append((float(s), str(r.get("task_id", "?")), str(r.get("variant_id") or "default"))) - for s, t, v in sorted(rows): - print(f" [{'ok' if s >= floor else 'BELOW'}] {v}/{t}: {s:.3f}") - worst = min(rows) if rows else None - ok = worst is not None and worst[0] >= floor - if ok: - print(f"score gate passed: lowest {worst[0]:.3f} >= floor {floor:.3f}") - elif worst is not None: - print(f"::error::score gate FAILED: lowest {worst[0]:.3f} ({worst[2]}/{worst[1]}) < floor {floor:.3f}") - else: - print(f"::error::score gate: no scored tasks in run.json to check against floor {floor:.3f}") - sys.exit(0 if ok else 1) - PY - fi - fi - - # Combine: coder-eval's own failure OR the score floor fails the step. - if [ "$CODE" -ne 0 ]; then - exit "$CODE" - fi - [ "$GATE" -eq 0 ] || exit 1 + exit "$CODE" diff --git a/docs/CI_GATE.md b/docs/CI_GATE.md index 281f833a..5f588fd6 100644 --- a/docs/CI_GATE.md +++ b/docs/CI_GATE.md @@ -1,8 +1,7 @@ --- description: >- Run Coder Eval as a CI gate — the coder_eval GitHub Action from the Actions - Marketplace, JUnit XML output for test-report ingestion, and an optional - per-task score floor. + Marketplace, and JUnit XML output for test-report ingestion. --- # CI Gate: GitHub Action & JUnit reports @@ -10,10 +9,9 @@ description: >- Coder Eval ships a **packaged CI gate**: a composite GitHub Action — on the Actions Marketplace as [**coder_eval**](https://github.com/marketplace/actions/coder_eval) — that -installs the CLI, runs your tasks, emits a JUnit XML report, appends the run -summary to the job summary, and fails the build on any task/gate failure. This -page is the reference for the Action and the JUnit output. For a walkthrough -(including a hand-rolled workflow), see +installs the CLI, runs your tasks, emits a JUnit XML report, and fails the build +on any task failure. This page is the reference for the Action and the JUnit +output. For a walkthrough (including a hand-rolled workflow), see [Tutorial 02 — Running Coder Eval in CI](tutorials/02-ci-pipeline.md). ## The GitHub Action @@ -30,8 +28,10 @@ repo path — there is no Marketplace install step: - uses: UiPath/coder_eval@v0 # …then run the gate (@v1 once 1.0.0 ships; @vX.Y.Z pins exactly) with: - tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml - model: claude-sonnet-5 + args: | + tests/tasks/**/*.yaml + --model + claude-sonnet-5 env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` @@ -44,48 +44,119 @@ those steps for your own agent's runtime as needed. ### Inputs +Eight, and **none of them is a `coder-eval run` flag**. GitHub silently *ignores* +an input the referenced tag does not define, so a forwarding input that is +mistyped or newer than your pin produces a run that measured something else and +still exits 0, where a wrong CLI flag is a hard error. Every flag goes through +`args`, and an input exists only where the action does something with the value +besides pass it along. + | Input | Default | Purpose | | --- | --- | --- | -| `tasks` | — | Task YAML path(s)/glob(s) passed to `coder-eval run`. Effectively required — see below. | -| `tags` | — | Only run tasks matching these comma-separated tags (`--tags`). | -| `model` | — | Override agent model for all tasks (`--model`). | -| `extra-args` | — | Extra args appended verbatim to `coder-eval run` (`--experiment`, `-D …`, `--exclude-tags`, …). Trusted caller input. | +| `args` | — | Everything for `coder-eval run` — task paths/globs and every flag — one argument per line, appended verbatim. See below. | | `version` | pinned release | `coder-eval` version to install from PyPI, or `local` to install from the action checkout. | -| `run-dir` | `runs/ci` | Run directory (`--run-dir`). | -| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit XML report. | -| `step-summary` | `true` | Append `run.md` to the GitHub job summary. | +| `extras` | — | Comma-separated `coder-eval` extras, composed into the install requirement (`codex`, `antigravity,litellm`). | +| `extra-packages` | — | Extra requirements installed into `coder-eval`'s environment (`uv tool install --with`), one per line. | +| `install-flags` | — | Flags for `uv tool install`, one per line (`--prerelease=allow`, `--extra-index-url …`). | | `env` | — | Credential/backend passthrough (see below). | -| `minimum-task-score` | *(off)* | Optional strict per-task score floor (see below). | +| `working-directory` | `.` | Directory every step of the action runs in — see below. | +| `run-dir` | `runs/ci` | Run directory (`--run-dir`). Also where the reports are written. | + +#### Writing `args` + +**One argument per line, appended verbatim.** No word splitting, no pathname +expansion. A flag and its value are **two lines**, or one line in `=` form: + +```yaml +args: | + tests/tasks/**/*.yaml + --tags + smoke + --model=claude-sonnet-5 + -D + sandbox.docker.env_passthrough_extra=[AUTH_TOKEN,BASE_URL] +``` + +A flag and value sharing a line arrive as a single malformed token, which the CLI +rejects. Blank lines, `#` comments and surrounding whitespace are ignored. + +Verbatim is the point: a bracketed `-D` value is a bash character class, so any +input that split on whitespace would survive only until a file in the working +directory happened to match and silently rewrote the list. + +Task globs are handed to the CLI **unexpanded**, and it expands them itself: + +- **`**` works.** `tests/tasks/**/*.yaml` is recursive, no `globstar` needed. +- **A glob matching nothing exits 1** with `No task files found!`, rather than + reaching the CLI as a literal path or vanishing. +- **Omitting `args` entirely does not run your suite.** Zero-argument discovery + resolves against `tasks/` relative to the working directory. Pass your paths. + +#### Extras and plugins (`extras`, `extra-packages`, `install-flags`) + +The action installs the CLI with `uv tool install`, which builds an isolated +environment whose shims **shadow** anything else named `coder-eval` on `PATH`, so +pre-installing your own copy beside it does not work. These inputs exist for that +reason. -#### Writing the `tasks` glob +`extras` is composed into the requirement string, so agent extras land in the +environment the action actually invokes: -Always pass `tasks` explicitly, and spell out each depth you actually have: +```yaml +extras: codex # -> coder-eval[codex]== +``` + +`extra-packages` adds requirements *into* that same environment, one per line — +a PEP 508 specifier or a local path. This is how a `coder-eval` plugin +distributed outside this repo becomes discoverable, since an entry point is only +found when the plugin shares a virtualenv with its host: ```yaml -tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml +extra-packages: | + ./vendor/my-coder-eval-plugin + some-published-plugin>=1.2 ``` -Three sharp edges make that worth the words: +`install-flags` passes resolver flags through, one per line, for what the install +needs and the action does not model: + +```yaml +install-flags: | + --prerelease=allow + --extra-index-url + https://my-private-index.example/simple +``` -- **Omitting `tasks` does not run everything.** The value is shell-expanded into the - `coder-eval run` argument list, so an empty one invokes the CLI with no paths — and - zero-argument discovery resolves against the *installed package's* location, not your - checkout. It finds nothing and exits 1. -- **Do not use `**`.** The expansion happens with `globstar` off, so - `tests/tasks/**/*.yaml` collapses to `tests/tasks/*/*.yaml` and **silently drops every - top-level task** — the gate goes green having never run them. -- **Only list depths that match.** `nullglob` is off too, so a pattern matching nothing - reaches the CLI verbatim and fails the run with - `Error: Task file not found: tests/tasks/*/*/*.yaml`. +#### Running from a subdirectory (`working-directory`) -An explicit file list is always safe, and is the better choice for a small suite. +A suite under `tests/` needs the run to happen there, and GitHub rejects +`working-directory:` on a `uses:` step — a job-level `defaults.run` does not reach +inside a composite either. This input is the way in. It applies to **every** step +the action runs, so `run-dir`, the task paths in `args` and relative +`extra-packages` entries all resolve against it. The `run-dir` output is reported +exactly as passed, so a relative one is relative to that directory, not to the +job's default cwd a later step reads it from. ### Outputs | Output | Description | | --- | --- | -| `run-dir` | The run directory containing `run.json` / `run.md`. | -| `junit-path` | Path to the written JUnit XML report. | +| `run-dir` | The run directory, as passed, containing `run.json` / `run.md`. | +| `junit-path` | The JUnit XML report, at `/junit.xml`. | +| `run-md-path` | The markdown run report, at `/run.md`. | + +There is no `junit-path` **input**: the report belongs with the run it describes, +and every consumer that had the choice put it there anyway. Nor does the action +append the report to `$GITHUB_STEP_SUMMARY` — a consumer that must redact it first +cannot undo a write that already happened, so that write is yours to make: + +```yaml +- id: eval + uses: UiPath/coder_eval@v0 + with: { args: "tests/tasks/**/*.yaml" } +- if: always() + run: cat "${{ steps.eval.outputs.run-md-path }}" >> "$GITHUB_STEP_SUMMARY" +``` ### Credentials via `env` @@ -99,7 +170,7 @@ values from repository secrets — never inline a secret literal. ```yaml - uses: UiPath/coder_eval@v0 with: - tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml + args: tests/tasks/**/*.yaml env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} API_BACKEND=direct @@ -111,16 +182,6 @@ vars, `GEMINI_API_KEY` for Antigravity, `EVALBOARD_*`, plugin paths, etc. See th per-agent guides ([Claude Code](agents/CLAUDE_CODE.md) · [Codex](agents/CODEX.md) · [Antigravity](agents/ANTIGRAVITY.md)) for what each backend needs. -### The score floor (`minimum-task-score`) - -An **additional** gate on top of `coder-eval`'s own exit code. Set a float in -`[0.0, 1.0]` and the step fails if **any** scored task, in any variant, has a -`weighted_score` below it — *or* if `coder-eval` itself exits non-zero (both -verdicts surface). It reads the always-written `run.json` spine -(`task_results[*].weighted_score`), so it works for plain and experiment runs -alike. Errored tasks (null score) are left to `coder-eval`'s exit code; a -malformed/`NaN` score fails closed. Empty (the default) disables the floor. - ### Security Evaluated tasks execute agent-generated code. **Do not** run this action under diff --git a/docs/index.md b/docs/index.md index 6960f0e3..5f22ce7e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -87,7 +87,7 @@ New here? Start with **[Tutorial 01 — Your First Evaluation](tutorials/01-firs | [Bring Your Own Dataset](DATASETS.md) | Fan a single task out over a dataset | | [Dialog Mode](DIALOG_MODE.md) | Evaluate agents in multi-turn conversation via a simulated user | | [Docker Isolation](DOCKER_ISOLATION.md) | The container sandbox driver, with custom images | -| [CI Gate & GitHub Action](CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, score floor | +| [CI Gate & GitHub Action](CI_GATE.md) | Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports | | [Claude Code Plugin](PLUGIN.md) | Install the Claude Code plugin — author, run, and analyze suites from inside the agent | | [Extending Coder Eval](EXTENDING.md) | Author a custom agent, criterion, or model pricing via the plugin SPI | | [Report Schema](REPORT_SCHEMA.md) | Field-level reference for run.json / variant.json / task.json | diff --git a/docs/llms.txt b/docs/llms.txt index 8e30b514..9dc3865f 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -35,7 +35,7 @@ and A/B plumbing. - [Bring Your Own Dataset](https://coder-eval.com/docs/datasets): Fan a single task out over a dataset - [Dialog Mode](https://coder-eval.com/docs/dialog-mode): Evaluate agents in multi-turn conversation via a simulated user - [Docker Isolation](https://coder-eval.com/docs/docker-isolation): The container sandbox driver, with custom images -- [CI Gate & GitHub Action](https://coder-eval.com/docs/ci-gate): Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, score floor +- [CI Gate & GitHub Action](https://coder-eval.com/docs/ci-gate): Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports - [Claude Code Plugin](https://coder-eval.com/docs/plugin): Install the Claude Code plugin — author, run, and analyze suites from inside the agent - [Extending Coder Eval](https://coder-eval.com/docs/extending): Author a custom agent, criterion, or model pricing via the plugin SPI - [Report Schema](https://coder-eval.com/docs/report-schema): Field-level reference for run.json / variant.json / task.json diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index a55507de..f6cb76c9 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -163,8 +163,7 @@ jobs: The five steps above spell out the mechanics, but Coder Eval also ships a composite action — on the Marketplace as [**coder_eval**](https://github.com/marketplace/actions/coder_eval) — that -bundles install + run + JUnit report + job-summary + fail-on-failure into one -step: +bundles install + run + JUnit report + fail-on-failure into one step: ```yaml - uses: actions/setup-node@v4 # the claude-code agent needs the Claude CLI… @@ -173,8 +172,10 @@ step: - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) with: - tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml - model: claude-sonnet-5 + args: | + tests/tasks/**/*.yaml + --model + claude-sonnet-5 env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` diff --git a/evalboard/README.md b/evalboard/README.md index fde55a10..8c3f6d84 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -107,21 +107,29 @@ and bootstrap CIs). ## Sources -A **source** is one blob container of runs, surfaced as its own tab -(`lib/sources.ts`). One deployment serves all of them — the container is a -runtime dimension threaded through the data layer as a trailing +A **source** is one blob container of runs (`lib/sources.ts`), usually surfaced as +its own tab. One deployment serves all of them — the container is a runtime +dimension threaded through the data layer as a trailing `source: Source = DEFAULT_SOURCE` parameter, not a build-time env var: | Source | Container | Surface | |--------|-----------|---------| | `skills` (default) | `runs` | Everything not listed below | | `scribe` | `aria-runs` | `/scribe` | +| `gha` | `runs-gha` | none — direct links only | Non-default sources are selected by a `?src=` query param, which every run-scoped page and API route reads. An absent or unrecognised `src` resolves to the default source (`sourceById` coerces rather than throwing, so a stray param in a shared link degrades to the skills dashboard instead of an error page). +**A source need not have a tab.** `NAV` in `app/layout.tsx` is a hardcoded array +and does not iterate `SOURCES`, so `gha` is registered and deliberately unlisted: +ad-hoc runs uploaded by UiPath/skills' `run-coder-eval` dispatch, reachable only +by the direct link in the GitHub run summary, expiring after 14 days under the +storage account's `expire-runs-gha-14d` rule. Registration is still mandatory — +`sourceById` is the only path by which a container becomes reachable at all. + Two invariants worth preserving if you add a source: - **Run ids are only unique within a container.** Every suite names runs @@ -136,7 +144,11 @@ Two invariants worth preserving if you add a source: `listRunIdsInWindow` filter on `parseRunIdDate`, so such runs surface only in the ad-hoc section. A new source's page therefore needs its OWN `getAdhocRunListing` section, or ad-hoc uploads to that container land - nowhere reachable. + nowhere reachable — unless it is unlisted by design, like `gha`, where nothing + enumerates and `app/runs/[id]` reads by id on demand. Note also that + `getAdhocRunListing` loads per-run metadata for **every** non-date-shaped id in + the container before truncating to the display limit, so a source expecting a + steady stream of ad-hoc uploads needs its own container, not a prefix in `runs`. - **Local mode is per-source too.** `listRunIds` resolves `runsDirFor(RUNS_DIR, source)` when `EVALBOARD_LOCAL_RUNS_DIR` is set, so `/scribe` reads `-scribe`. Listing off the bare local dir instead — diff --git a/evalboard/lib/__tests__/source-isolation.test.ts b/evalboard/lib/__tests__/source-isolation.test.ts index 6def25ac..6e6ca133 100644 --- a/evalboard/lib/__tests__/source-isolation.test.ts +++ b/evalboard/lib/__tests__/source-isolation.test.ts @@ -2,7 +2,7 @@ import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { SCRIBE_SOURCE } from "@/lib/sources"; +import { GHA_SOURCE, SCRIBE_SOURCE, SOURCES, runsDirFor } from "@/lib/sources"; // The invariant these tests pin: run ids are only unique WITHIN a container. // Both suites name runs `YYYY-MM-DD_HH-MM-SS`, so a same-day skills run and @@ -86,7 +86,11 @@ afterEach(async () => { else process.env[k] = savedEnv[k]; } await fs.rm(localDir, { recursive: true, force: true }); - await fs.rm(`${localDir}-scribe`, { recursive: true, force: true }); + // Every sibling, derived rather than listed: a source added without a matching + // line here would leak its tree into the next test's listing assertions. + for (const s of SOURCES) { + await fs.rm(runsDirFor(localDir, s), { recursive: true, force: true }); + } }); describe("reader-layer source isolation", () => { @@ -167,6 +171,47 @@ describe("reader-layer source isolation", () => { expect(await listRunIds(SCRIBE_SOURCE)).toEqual([RUN_ID, scribeOnly]); }); + // The gha source has no listing page and no tab, so `readRunSummary` by id is + // the ONLY reader it ever exercises — and a link pasted out of a GitHub run + // summary is the only way anyone arrives. If that read resolved against the + // default container it would render the skills nightly under the dispatcher's + // run id: a plausible-looking page, not a 404. Nothing else would catch it. + test("readRunSummary is scoped for the unlisted gha source too", async () => { + const ghaRun = path.join(runsDirFor(localDir, GHA_SOURCE), RUN_ID); + await fs.mkdir(ghaRun, { recursive: true }); + await fs.writeFile( + path.join(ghaRun, "run.json"), + runJson({ + tasksRun: 1, + tasksSucceeded: 1, + startTime: "2026-08-14T23:00:00Z", + }), + ); + + const { readRunSummary } = await loadRuns(); + const gha = await readRunSummary(RUN_ID, GHA_SOURCE); + expect(gha?.tasksRun).toBe(1); + // The skills tree seeded in beforeEach has 100 under the same id. + expect((await readRunSummary(RUN_ID))?.tasksRun).toBe(100); + + // And an id present only in gha must not render out of `runs`. + const ghaOnly = "2026-08-14_23-30-00"; + await fs.mkdir(path.join(runsDirFor(localDir, GHA_SOURCE), ghaOnly), { + recursive: true, + }); + await fs.writeFile( + path.join(runsDirFor(localDir, GHA_SOURCE), ghaOnly, "run.json"), + runJson({ + tasksRun: 3, + tasksSucceeded: 3, + startTime: "2026-08-14T23:30:00Z", + }), + ); + const { readRunSummary: fresh } = await loadRuns(); + expect(await fresh(ghaOnly, GHA_SOURCE)).not.toBeNull(); + expect(await fresh(ghaOnly)).toBeNull(); + }); + test("latestRunId is per source", async () => { const { latestRunId } = await loadRuns(); expect(await latestRunId()).toBe(RUN_ID); diff --git a/evalboard/lib/__tests__/sources.test.ts b/evalboard/lib/__tests__/sources.test.ts index 9465c525..d13592ef 100644 --- a/evalboard/lib/__tests__/sources.test.ts +++ b/evalboard/lib/__tests__/sources.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "vitest"; import { DEFAULT_SOURCE, + GHA_SOURCE, SCRIBE_SOURCE, SKILLS_SOURCE, SOURCES, @@ -19,6 +20,14 @@ describe("source registry", () => { expect(SCRIBE_SOURCE.container).not.toBe(SKILLS_SOURCE.container); }); + test("gha reads its own container, so the 14-day expiry rule cannot reach nightly history", () => { + // The storage account carries a lifecycle rule (`expire-runs-gha-14d`, + // prefixMatch `runs-gha/`) that DELETES blobs. Sharing a container with + // the skills nightly would put months of history behind that rule. + expect(GHA_SOURCE.container).toBe("runs-gha"); + expect(GHA_SOURCE.container).not.toBe(SKILLS_SOURCE.container); + }); + test("every source has a distinct id and container", () => { const ids = SOURCES.map((s) => s.id); const containers = SOURCES.map((s) => s.container); @@ -41,6 +50,17 @@ describe("sourceById", () => { expect(sourceById("skills")).toBe(SKILLS_SOURCE); }); + // The gha source is reachable ONLY by `?src=gha` on a link pasted from a + // GitHub run summary — there is no tab and no listing to arrive from. So the + // id in that emitted link and the id in the registry are a two-place + // agreement with no UI in between to reveal a mismatch, and the coercion + // asserted below turns a typo in either into the wrong container's data + // rather than an error. This is the assertion that fails instead. + test("resolves the unlisted gha id to the gha container", () => { + expect(sourceById("gha")).toBe(GHA_SOURCE); + expect(sourceById("gha").container).toBe("runs-gha"); + }); + // Coercing rather than throwing is deliberate: a stray ?src= in a shared URL // should show the default dashboard, not an error page. The tradeoff is that a // TYPO'd source silently shows skills data — asserted here so that behaviour @@ -99,6 +119,30 @@ describe("runsDirFor", () => { // and refresh-button), so a Node builtin import here fails `next build` with // UnhandledSchemeError — which tsc and vitest both happily pass. Catch it here so // the failure surfaces in a fast test rather than only in the production build. +// The gha source is registered but deliberately absent from the header nav: a +// run is reachable only by its direct link from the GitHub run that produced it. +// That is what lets it skip a listing page — and evalboard/README.md warns that a +// source WITH a tab needs its own `getAdhocRunListing` section or its uploads land +// nowhere reachable. So adding the tab without the listing is the silent failure +// this pins. If a tab is genuinely wanted, add the listing section first, then +// delete this test. +describe("unlisted sources", () => { + test("gha is registered but has no nav tab", async () => { + const { readFile } = await import("node:fs/promises"); + const { join } = await import("node:path"); + const layout = await readFile( + join(process.cwd(), "app/layout.tsx"), + "utf-8", + ); + expect(SOURCES).toContain(GHA_SOURCE); + expect(layout).not.toContain(`"/${GHA_SOURCE.id}"`); + // Guard against the pin rotting the other way: if NAV ever starts + // iterating SOURCES, registration alone would create the tab and the + // href check above would keep passing while the tab appeared. + expect(layout).not.toMatch(/NAV[\s\S]{0,200}SOURCES/); + }); +}); + describe("client-safety", () => { test("sources.ts imports no Node builtins", async () => { const { readFile } = await import("node:fs/promises"); diff --git a/evalboard/lib/sources.ts b/evalboard/lib/sources.ts index 353b23ee..68ddf52b 100644 --- a/evalboard/lib/sources.ts +++ b/evalboard/lib/sources.ts @@ -43,7 +43,26 @@ export const SCRIBE_SOURCE: Source = { container: "aria-runs", }; -export const SOURCES: readonly Source[] = [SKILLS_SOURCE, SCRIBE_SOURCE]; +// Ad-hoc runs uploaded by UiPath/skills' `run-coder-eval` workflow_dispatch, so a +// debug run has a shareable link instead of only a downloadable artifact. +// +// DELIBERATELY UNLISTED: registered here but absent from `NAV` in app/layout.tsx, +// so there is no tab and nothing enumerates it — a run is reachable only by the +// direct link from the GitHub run that produced it, which is why the README's +// `getAdhocRunListing` rule does not apply. Registration is still mandatory: +// `sourceById` COERCES an unknown id to DEFAULT_SOURCE rather than throwing, so +// without this entry `?src=gha` would read the nightly's container and 404. +// +// Its own container, not `runs` with a prefix: getAdhocRunListing loads per-run +// metadata for every non-date-shaped id before truncating, and the 14-day expiry +// rule (`expire-runs-gha-14d`) must never reach nightly history. +export const GHA_SOURCE: Source = { + id: "gha", + label: "Ad-hoc (GH)", + container: "runs-gha", +}; + +export const SOURCES: readonly Source[] = [SKILLS_SOURCE, SCRIBE_SOURCE, GHA_SOURCE]; /** Every surface that doesn't opt into a source reads the skills nightly. */ export const DEFAULT_SOURCE = SKILLS_SOURCE; diff --git a/mkdocs.yml b/mkdocs.yml index 44f2c734..db2d2728 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,7 +89,7 @@ extra: DATASETS.md: "Fan a single task out over a dataset" DIALOG_MODE.md: "Evaluate agents in multi-turn conversation via a simulated user" DOCKER_ISOLATION.md: "The container sandbox driver, with custom images" - CI_GATE.md: "Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, score floor" + CI_GATE.md: "Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports" PLUGIN.md: "Install the Claude Code plugin — author, run, and analyze suites from inside the agent" EXTENDING.md: "Author a custom agent, criterion, or model pricing via the plugin SPI" REPORT_SCHEMA.md: "Field-level reference for run.json / variant.json / task.json" diff --git a/plugins/coder-eval/skills/ci/SKILL.md b/plugins/coder-eval/skills/ci/SKILL.md index 3d1b2729..5974e36a 100644 --- a/plugins/coder-eval/skills/ci/SKILL.md +++ b/plugins/coder-eval/skills/ci/SKILL.md @@ -1,5 +1,5 @@ --- -description: Generate a GitHub Actions workflow that runs a coder-eval suite as a CI gate or on a schedule, using the published composite action — with the agent runtime, credentials, JUnit output and a score floor wired correctly. +description: Generate a GitHub Actions workflow that runs a coder-eval suite as a CI gate or on a schedule, using the published composite action — with the agent runtime, credentials, JUnit output and the run reports wired correctly. disable-model-invocation: true allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"] --- @@ -12,7 +12,7 @@ The user's request is: `$ARGUMENTS` Find the repository's task tree by following `${CLAUDE_PLUGIN_ROOT}/reference/repo-layout.md`, and check whether `.github/workflows/` -exists. The paths you resolve here become the workflow's `tasks:` input in step 3 — that +exists. The paths you resolve here become `args:` entries in the workflow in step 3 — that input is written from discovery, never from a fixed guess. If there is no `.github/` directory at all, say that this skill targets GitHub Actions @@ -71,44 +71,46 @@ jobs: node-version: "20" - run: npm install -g @anthropic-ai/claude-code - - uses: UiPath/coder_eval@v0 + - id: eval + uses: UiPath/coder_eval@v0 with: - tasks: tasks/*.yaml - model: claude-haiku-4-5-20251001 - junit-path: runs/ci/junit.xml - step-summary: true - minimum-task-score: "0.7" + run-dir: runs/ci + args: | + tasks/**/*.yaml + --model + claude-haiku-4-5-20251001 env: | ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} + + - if: always() + run: cat "${{ steps.eval.outputs.run-md-path }}" >> "$GITHUB_STEP_SUMMARY" ``` -Adjust `model:` and the cron to the repository. Pin the action at `@v0`, the moving major +Adjust the model and the cron to the repository. Pin the action at `@v0`, the moving major tag. Then work through the four things the snippet cannot guess. -### `tasks:` — from discovery, and never with `**` +Note the shape of `args:`: the action promotes **none** of `coder-eval run`'s flags to a +named input, so task paths and flags all go there, one argument per line, with a flag and +its value on separate lines. There is no `tasks:`, `tags:` or `model:` input. -The value above is a placeholder for whatever step 1 discovered. Substituting it is not -just a rename, because **the action expands this input unquoted with `globstar` off**: -bash word-splits *and* pathname-expands it before coder-eval ever sees it. +### The task paths — from discovery -- **A recursive `**` glob silently loses tasks.** With `globstar` off, `a/**/*.yaml` - degrades to `a/*/*.yaml` — so a tree with `a/top.yaml` and `a/sub/deep.yaml` runs - `deep.yaml` only, and the gate passes while never testing `top.yaml`. Nothing reports - this. Do not write `**` here, and keep this paragraph next to whatever you do write, or - the next reader will "simplify" it back. -- **An unmatched glob is worse than a missing one.** `nullglob` is off too, so a pattern - matching nothing reaches the CLI as a literal string and hard-fails the whole run - (`Error: Task file not found: …`, exit 1). +The value above is a placeholder for whatever step 1 discovered. `args:` entries are +handed to the CLI **verbatim**: no word splitting, no pathname expansion. The CLI expands +the globs itself, which makes this simpler than it looks: -So emit **explicit per-depth globs, or an explicit file list** — and emit only the depths -that actually match when you write the workflow. Check first; a fixed ladder of depths -breaks any repository that does not happen to have tasks at every level. +- **`**` works.** `tasks/**/*.yaml` is genuinely recursive, so one pattern covers a tree + of any depth. No per-depth ladder, no `globstar` caveat. +- **A glob matching nothing fails loudly** with `No task files found!` and exit 1, rather + than reaching the CLI as a literal path. +- **One path per line.** Two globs are two lines, not one space-separated string, which + would arrive as a single malformed argument. -For a tree that happens to sit two levels deep, that looks like this — the paths are one -repository's, shown to make the shape concrete, not a value to copy: +So emit what step 1 found, one entry per line: ```yaml -tasks: tests/tasks/*.yaml tests/tasks/*/*.yaml +args: | + tests/tasks/**/*.yaml ``` ### `version:` — conditional on the repository's pin @@ -122,22 +124,22 @@ self-documenting. ### The experiment, if the suite runs through one -If the repository's suite resolves through an experiment, the workflow must pass it via -`extra-args` — again with the discovered path, not the illustrative one below: +If the repository's suite resolves through an experiment, the workflow must pass it in +`args` — two lines, and with the discovered path, not the illustrative one below: ```yaml -extra-args: "-e tests/experiments/default.yaml" +args: | + tests/tasks/**/*.yaml + -e + tests/experiments/default.yaml ``` -This is load-bearing rather than tidy: an experiment usually supplies `agent:` config, so +This matters rather than being tidy: an experiment usually supplies `agent:` config, so omitting it silently changes what the run measures — the gate and the local run stop being the same test. If the repository has **several** experiments, ask which one the gate should use; a CI gate quietly running the wrong experiment is precisely the failure this exists to prevent. -`extra-args` is a trusted input that is split on whitespace, so a path containing a space -is unsafe there. Choose paths without spaces rather than discovering this in CI. - ### Environment — including the skill source, if the suite is an activation suite If the resolved experiment or the tasks interpolate environment variables, pass them @@ -192,27 +194,23 @@ Never inline a key literal, and never commit one. If the repository has no ## Step 5 — Reports -- `junit-path:` writes a JUnit XML report, which GitHub and most test-report tooling - ingest to show per-task pass/fail. -- `step-summary: true` appends the run's markdown report to the job summary, so a - reviewer sees the scores without downloading anything. +The action reports three paths as outputs and writes no report anywhere itself: + +- `junit-path` — the JUnit XML at `/junit.xml`, which GitHub and most + test-report tooling ingest to show per-task pass/fail. +- `run-md-path` — the run's markdown report. Append it to the job summary, as the + snippet does, so a reviewer sees the scores without downloading anything. The action + deliberately does not do this for you: a workflow that has to redact the report first + cannot undo a write that already happened. +- `run-dir` — the whole run directory. Consider uploading the run directory as an artifact on failure so a failing gate can be analyzed with `/coder-eval:analyze` afterwards. -## Step 6 — Choose the floor - -`minimum-task-score` is a strict floor: **every** scored task, in every variant, must -reach it or the step fails. It sits on top of coder-eval's own exit code — the step fails -if either coder-eval fails or any task scores below the floor. Leave it empty to disable -it. - -Explain the tradeoff and let the user pick rather than choosing for them: a floor that is -too high makes the gate flaky (agents are nondeterministic), one that is too low never -catches anything. Suggest running the suite once, then setting the floor a little below -the observed minimum. +The step's exit code is coder-eval's own: non-zero on any failed task. There is no score +floor input; a suite that needs one gates on `run.json` in a following step. -## Step 7 — Warn about fork PRs, and explain the two hardening lines +## Step 6 — Warn about fork PRs, and explain the two hardening lines Evaluated tasks execute agent-generated code. Never run this under `pull_request_target` with secrets exposed to untrusted fork PRs — that combination diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 6a3acadb..9ef304cf 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -76,21 +76,34 @@ def expand_task_files(task_files: list[Path]) -> list[Path]: List of resolved task file paths Raises: - typer.Exit: If no task files are found + typer.Exit: If any pattern matches no task file """ all_task_files = [] + # Per-pattern, not just on the union. Accumulating and checking only the + # total meant one stale entry among several (a renamed or moved suite) + # silently ran the surviving subset and exited 0, so a CI gate reported + # green over tasks it never measured. A pattern the caller wrote is a + # pattern the caller expects to match something. + unmatched = [] for pattern in task_files: if pattern.is_file(): all_task_files.append(pattern) + continue + # Try as glob pattern (supports ** for recursive matching) + if pattern.is_absolute(): + matches = list(Path(pattern.anchor).glob(str(pattern.relative_to(pattern.anchor)))) else: - # Try as glob pattern (supports ** for recursive matching) - if pattern.is_absolute(): - all_task_files.extend(Path(pattern.anchor).glob(str(pattern.relative_to(pattern.anchor)))) - else: - all_task_files.extend(Path().glob(str(pattern))) - - if not all_task_files: + matches = list(Path().glob(str(pattern))) + if not matches: + unmatched.append(pattern) + all_task_files.extend(matches) + + # The union check still stands on its own: an empty `task_files` reaches here + # with nothing unmatched, and returning [] would run a zero-task suite green. + if unmatched or not all_task_files: console.print("[red]No task files found![/red]") + for pattern in unmatched: + console.print(f"[red] no match: {pattern}[/red]") raise typer.Exit(1) random.shuffle(all_task_files) diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py new file mode 100644 index 00000000..a5db4cd1 --- /dev/null +++ b/tests/test_action_inputs.py @@ -0,0 +1,499 @@ +"""Executable contract for the argv ``action.yml`` builds from its inputs. + +The composite action's two bash steps assemble two command lines — a ``uv tool +install`` and a ``coder-eval run`` — out of eight string inputs. Everything that +can go wrong there goes wrong *silently*: an extra dropped from the requirement +string installs a working CLI that is missing an agent, and a value mangled by +word splitting or pathname expansion reaches the CLI as a different value than +the workflow wrote, so the run measures something else and still exits 0. + +These tests therefore execute the shipped script rather than reimplementing it. +Each step's ``run:`` body is pulled straight out of ``action.yml`` and run under +bash with ``uv`` / ``coder-eval`` replaced by stubs that record their argv, so the +assertions are about the real text that ships to consumers. A rewrite of the +script that changes the resulting command line fails here even if it looks +equivalent. + +The design these tests pin: the action promotes NONE of ``coder-eval run``'s 21 +flags to a named input. Everything goes through ``args``, one argv entry per +line, appended verbatim. That is what makes a ``-D`` override whose value is a +bracketed list (``key=[A,B,C]`` — a bash character class) survive; the earlier +whitespace-split input silently rewrote it to one name whenever a file in the +working directory happened to match. +""" + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +ACTION_YML = REPO_ROOT / "action.yml" + +BASH = shutil.which("bash") + +# The run step needs every CE_* name defined (`set -u`), so each case supplies only +# what it varies. +RUN_ENV_DEFAULTS = { + "CE_ARGS": "", + "CE_RUN_DIR": "runs/ci", + "CE_ENV": "", +} + +INSTALL_ENV_DEFAULTS = { + "CE_VERSION": "9.9.9", + "CE_EXTRAS": "", + "CE_EXTRA_PACKAGES": "", + "CE_INSTALL_FLAGS": "", + "CE_ACTION_PATH": "/action-checkout", +} + + +def _step_script(step_name: str) -> str: + """The ``run:`` body of a named step, as it ships. + + Read from action.yml rather than duplicated here: a test holding its own copy + of the script asserts nothing about what consumers get. + """ + data = yaml.safe_load(ACTION_YML.read_text(encoding="utf-8")) + for step in data["runs"]["steps"]: + if step.get("name") == step_name: + return step["run"] + raise AssertionError(f"action.yml has no step named {step_name!r}") + + +def _stub(dir_: Path, name: str) -> Path: + """A fake executable recording its argv and its inherited ``CE_PROBE``, exit 0. + + Written in bash rather than Python on purpose. ``shell: bash`` on a Windows + runner is Git Bash, which rewrites arguments that look like absolute POSIX + paths on the way to a *native* Windows binary: a python-shebang stub gets + ``/action-checkout`` as ``C:/Program Files/Git/action-checkout``, and + switching that conversion off only moves the failure, because the shebang + launcher then cannot hand python its own script path either. A bash stub + never crosses that boundary, so argv arrives byte-for-byte everywhere. + + argv is recorded NUL-delimited instead of as JSON so a value carrying a + quote, a backslash or a space needs no escaping on the way out of bash. Each + invocation truncates the file; no test invokes the stub twice. + + ``CE_PROBE`` is how the env-passthrough test observes what the child actually + received: the passthrough exports into the step's own shell, so only a process + the script itself launches can report it. + """ + record = dir_ / "argv.bin" + # Forward slashes, not the native separator: the consumer is MSYS bash, which + # reads `C:/...` but not every backslash form. + quoted_dir = shlex.quote(str(dir_).replace("\\", "/")) + exe = dir_ / name + exe.write_text( + "#!/usr/bin/env bash\n" + f"d={quoted_dir}\n" + ': > "$d/argv.bin"\n' + 'for a in "$@"; do printf "%s\\0" "$a" >> "$d/argv.bin"; done\n' + 'printf "%s" "${CE_PROBE-}" > "$d/probe.txt"\n', + encoding="utf-8", + ) + exe.chmod(0o755) + return record + + +def _run(script: str, env: dict[str, str], *, cwd: Path, stub: str) -> tuple[int, list[str], str]: + """Execute ``script`` with ``stub`` shadowing the real binary; return (rc, argv, stderr+stdout).""" + bindir = cwd / "_stubbin" + bindir.mkdir(exist_ok=True) + record = _stub(bindir, stub) + + full_env = { + "PATH": f"{bindir}{os.pathsep}{os.environ['PATH']}", + "HOME": str(cwd), + "GITHUB_OUTPUT": str(cwd / "gh_output"), + "GITHUB_STEP_SUMMARY": str(cwd / "gh_summary"), + **env, + } + (cwd / "gh_output").touch() + (cwd / "gh_summary").touch() + + proc = subprocess.run( + [BASH, "-c", script], + cwd=cwd, + env=full_env, + capture_output=True, + text=True, + timeout=60, + ) + argv: list[str] = [] + if record.exists(): + # Trailing NUL terminates the last entry, so the split leaves an empty tail. + argv = [part.decode() for part in record.read_bytes().split(b"\0")[:-1]] + return proc.returncode, argv, proc.stdout + proc.stderr + + +@pytest.fixture(scope="module") +def install_script() -> str: + return _step_script("Install coder-eval") + + +@pytest.fixture(scope="module") +def run_script() -> str: + return _step_script("Run coder-eval") + + +def _install(script: str, tmp_path: Path, **overrides: str) -> tuple[int, list[str], str]: + return _run(script, {**INSTALL_ENV_DEFAULTS, **overrides}, cwd=tmp_path, stub="uv") + + +def _coder_eval(script: str, tmp_path: Path, **overrides: str) -> tuple[int, list[str], str]: + return _run(script, {**RUN_ENV_DEFAULTS, **overrides}, cwd=tmp_path, stub="coder-eval") + + +def _outputs(tmp_path: Path) -> dict[str, str]: + """The step's `$GITHUB_OUTPUT` writes, parsed.""" + text = (tmp_path / "gh_output").read_text(encoding="utf-8") + return dict(line.split("=", 1) for line in text.splitlines() if "=" in line) + + +class TestSharedParser: + # `install-flags`, `extra-packages` and `args` are all "one entry per line, + # verbatim". One implementation, copied into both step scripts because they + # are separate bash processes. Copies drift; this is what stops them. + def test_clean_lines_is_byte_identical_in_both_steps(self, install_script, run_script): + pattern = re.compile(r"^clean_lines\(\) \{\n.*?^\}\n", re.S | re.M) + a = pattern.search(install_script) + b = pattern.search(run_script) + assert a and b, "clean_lines() is missing from one of the step scripts" + assert a.group(0) == b.group(0), "the two clean_lines() copies have drifted apart" + + +class TestInstallSpec: + def test_defaults_install_the_pinned_release(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path) + assert rc == 0, out + assert argv == ["tool", "install", "coder-eval==9.9.9"] + + def test_local_installs_the_action_checkout(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path, CE_VERSION="local") + assert rc == 0, out + assert argv == ["tool", "install", "/action-checkout"] + + # Extras must land in the requirement string, not a follow-up install: the tool + # environment's shims shadow every other coder-eval on PATH, so an extra added + # beside it is never imported by the CLI the action goes on to invoke. + def test_extras_are_composed_into_the_requirement(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path, CE_EXTRAS="codex") + assert rc == 0, out + assert argv == ["tool", "install", "coder-eval[codex]==9.9.9"] + + def test_extras_compose_onto_a_local_install_too(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path, CE_VERSION="local", CE_EXTRAS="codex") + assert rc == 0, out + assert argv == ["tool", "install", "/action-checkout[codex]"] + + def test_multiple_extras_stay_comma_joined(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path, CE_EXTRAS="antigravity,litellm") + assert rc == 0, out + assert argv == ["tool", "install", "coder-eval[antigravity,litellm]==9.9.9"] + + @pytest.mark.parametrize( + "bad", + [ + "codex;rm -rf /", # shell metacharacters + "codex litellm", # space instead of comma + "codex,", # trailing comma + ",codex", # leading comma + "-codex", # must start alphanumeric + "$(id)", # command substitution + ], + ) + def test_malformed_extras_fail_before_installing(self, install_script, tmp_path, bad): + rc, argv, out = _install(install_script, tmp_path, CE_EXTRAS=bad) + assert rc != 0 + assert argv == [], "install ran despite malformed extras" + assert "extras must be a comma-separated list" in out + + def test_extra_packages_become_one_with_flag_each(self, install_script, tmp_path): + rc, argv, out = _install(install_script, tmp_path, CE_EXTRA_PACKAGES="./plugin-a\n../plugin-b\n") + assert rc == 0, out + assert argv == [ + "tool", + "install", + "--with", + "./plugin-a", + "--with", + "../plugin-b", + "coder-eval==9.9.9", + ] + + # A specifier can contain `[`, `]`, `>` and `=`; a local path can contain a + # space. None of those survive an unquoted expansion. + def test_extra_package_specifiers_survive_verbatim(self, install_script, tmp_path): + specs = ["coder-eval-uipath[dev]>=1.2,<2.0", "/opt/my plugin", "pkg!=0.2.144"] + rc, argv, out = _install(install_script, tmp_path, CE_EXTRA_PACKAGES="\n".join(specs)) + assert rc == 0, out + expected = ["tool", "install"] + for spec in specs: + expected += ["--with", spec] + assert argv == [*expected, "coder-eval==9.9.9"] + + def test_install_flags_are_appended_one_per_line(self, install_script, tmp_path): + rc, argv, out = _install( + install_script, + tmp_path, + CE_INSTALL_FLAGS="--prerelease=allow\n--extra-index-url\nhttps://example.test/simple\n", + ) + assert rc == 0, out + assert argv == [ + "tool", + "install", + "--prerelease=allow", + "--extra-index-url", + "https://example.test/simple", + "coder-eval==9.9.9", + ] + + # Install flags precede --with and the requirement: uv accepts flags anywhere, + # but a stable order is what makes these assertions meaningful at all. + def test_install_flags_precede_extra_packages(self, install_script, tmp_path): + rc, argv, out = _install( + install_script, + tmp_path, + CE_INSTALL_FLAGS="--prerelease=allow", + CE_EXTRA_PACKAGES="./plugin", + ) + assert rc == 0, out + assert argv == [ + "tool", + "install", + "--prerelease=allow", + "--with", + "./plugin", + "coder-eval==9.9.9", + ] + + @pytest.mark.parametrize("var", ["CE_EXTRA_PACKAGES", "CE_INSTALL_FLAGS"]) + def test_blank_lines_comments_and_padding_are_ignored(self, install_script, tmp_path, var): + rc, argv, out = _install(install_script, tmp_path, **{var: "\n ./plugin-a \n\n# a comment\n\t./plugin-b\n\n"}) + assert rc == 0, out + assert "./plugin-a" in argv and "./plugin-b" in argv + assert not any("comment" in a for a in argv) + assert not any(a.strip() != a for a in argv), f"an entry kept its padding: {argv}" + + +class TestRunArgs: + def test_baseline_argv(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path) + assert rc == 0, out + assert argv == ["run", "--run-dir", "runs/ci", "--junit-xml", "runs/ci/junit.xml"] + + # There is no `tasks` input: task paths and globs are `args` entries like any + # other. The CLI expands globs itself (`expand_task_files`), so passing them + # unexpanded is not a loss — and it exits 1 when nothing matches, where a + # shell would have silently passed the literal through. + def test_task_globs_are_ordinary_args(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS="skills/**/*.yaml\nrpa/*.yaml\n") + assert rc == 0, out + assert argv[-2:] == ["skills/**/*.yaml", "rpa/*.yaml"] + + def test_args_are_appended_one_entry_per_line(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS="--tags\nsmoke\n--type\ncodex\n") + assert rc == 0, out + assert argv == [ + "run", + "--run-dir", + "runs/ci", + "--junit-xml", + "runs/ci/junit.xml", + "--tags", + "smoke", + "--type", + "codex", + ] + + def test_args_blank_lines_comments_and_padding_are_ignored(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS="\n -v \n\n# note\n\t--stream\n\n") + assert rc == 0, out + assert argv[-2:] == ["-v", "--stream"] + + # THE motivating case. `[...]` is a bash character class, so a whitespace-split + # input drops list members whenever a file in the working directory matches. + # A file is planted here so the test would fail under any implementation that + # word-splits or glob-expands. + def test_bracketed_override_survives_verbatim(self, run_script, tmp_path): + (tmp_path / "agent.allowed_tools=Read").touch() + override = "agent.allowed_tools=[Read,Write,Bash]" + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS=f"-D\n{override}\n") + assert rc == 0, out + assert argv[-2:] == ["-D", override] + + def test_values_with_spaces_survive(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS="--model\nmodel with spaces\n") + assert rc == 0, out + assert argv[-2:] == ["--model", "model with spaces"] + + def test_run_dir_reaches_the_cli(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_RUN_DIR="/tmp/runs") + assert rc == 0, out + assert argv[:5] == ["run", "--run-dir", "/tmp/runs", "--junit-xml", "/tmp/runs/junit.xml"] + + +class TestOutputs: + # There is no `junit-path` input. The report belongs with the run it + # describes, and both consumers that had the choice already put it there. + def test_report_paths_are_derived_from_run_dir(self, run_script, tmp_path): + rc, _, out = _coder_eval(run_script, tmp_path, CE_RUN_DIR="/tmp/runs") + assert rc == 0, out + assert _outputs(tmp_path) == { + "run-dir": "/tmp/runs", + "junit-path": "/tmp/runs/junit.xml", + "run-md-path": "/tmp/runs/run.md", + } + + def test_a_trailing_slash_does_not_double_the_separator(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_RUN_DIR="runs/ci/") + assert rc == 0, out + o = _outputs(tmp_path) + assert o["junit-path"] == "runs/ci/junit.xml" + assert o["run-md-path"] == "runs/ci/run.md" + # --run-dir is forwarded as given; only the derived paths are normalised. + assert argv[2] == "runs/ci/" + + # run.json/run.md are written even on a red run, so a consumer uploading + # artifacts after a failure needs the paths. + def test_outputs_are_written_before_a_failing_exit(self, run_script, tmp_path): + bindir = tmp_path / "_stubbin" + bindir.mkdir() + _stub(bindir, "coder-eval") + (bindir / "coder-eval").write_text("#!/usr/bin/env bash\nexit 3\n", encoding="utf-8") + (bindir / "coder-eval").chmod(0o755) + (tmp_path / "gh_output").touch() + proc = subprocess.run( + [BASH, "-c", _step_script("Run coder-eval")], + cwd=tmp_path, + env={ + "PATH": f"{bindir}{os.pathsep}{os.environ['PATH']}", + "HOME": str(tmp_path), + "GITHUB_OUTPUT": str(tmp_path / "gh_output"), + "GITHUB_STEP_SUMMARY": str(tmp_path / "gh_summary"), + **RUN_ENV_DEFAULTS, + }, + capture_output=True, + text=True, + timeout=60, + ) + assert proc.returncode == 3, "the step must exit with coder-eval's own code" + assert _outputs(tmp_path)["run-dir"] == "runs/ci" + + # The action no longer appends run.md to the job summary: a consumer that has + # to redact the report first cannot undo a write that already happened. + def test_nothing_is_written_to_the_job_summary(self, run_script, tmp_path): + (tmp_path / "runs" / "ci").mkdir(parents=True) + (tmp_path / "runs" / "ci" / "run.md").write_text("# report\n", encoding="utf-8") + rc, _, out = _coder_eval(run_script, tmp_path) + assert rc == 0, out + assert (tmp_path / "gh_summary").read_text(encoding="utf-8") == "" + + +class TestEnvPassthrough: + def test_env_reaches_the_child(self, run_script, tmp_path): + rc, _, out = _coder_eval(run_script, tmp_path, CE_ENV="CE_PROBE=hello\n") + assert rc == 0, out + assert (tmp_path / "_stubbin" / "probe.txt").read_text(encoding="utf-8") == "hello" + + # The two multi-line inputs are parsed by different loops (env values are not + # right-trimmed, because a value is the caller's data). Neither may consume + # the other's content. + def test_args_and_env_do_not_bleed_into_each_other(self, run_script, tmp_path): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ARGS="--stream\n", CE_ENV="CE_PROBE=v\n") + assert rc == 0, out + assert argv[-1] == "--stream" + assert "CE_PROBE=v" not in argv + assert (tmp_path / "_stubbin" / "probe.txt").read_text(encoding="utf-8") == "v" + + def test_a_value_containing_equals_is_kept_whole(self, run_script, tmp_path): + rc, _, out = _coder_eval(run_script, tmp_path, CE_ENV="CE_PROBE=a=b=c\n") + assert rc == 0, out + assert (tmp_path / "_stubbin" / "probe.txt").read_text(encoding="utf-8") == "a=b=c" + + @pytest.mark.parametrize( + ("bad", "message"), + [ + ("NOEQUALS", "is not NAME=VALUE"), + ("2LEADING_DIGIT=x", "invalid name"), + ("has space=x", "invalid name"), + ("has-dash=x", "invalid name"), + ], + ) + def test_malformed_env_fails_before_running(self, run_script, tmp_path, bad, message): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ENV=bad) + assert rc != 0 + assert argv == [], "coder-eval ran despite a malformed env entry" + assert message in out + + # A caller who omits `NAME=` would otherwise have the secret printed verbatim + # in the error, and GitHub only masks values it already knows are secrets. + def test_a_malformed_entry_is_reported_by_position_not_by_value(self, run_script, tmp_path): + rc, _, out = _coder_eval(run_script, tmp_path, CE_ENV="s3cr3t-token-value") + assert rc != 0 + assert "s3cr3t-token-value" not in out + assert "entry #1" in out + + # An `env` value carrying a newline splits into a second entry, because the + # loop is line-based. That used to be an argv-rewrite: the pairs were + # `export`ed into the step's own shell, which is where CE_ARGS and + # CE_RUN_DIR are read from AFTER the loop. They are collected and handed to + # `env` now, so the injected entry reaches the child as data and nothing + # else. Reachable without a hostile author: any interpolated value or a + # rotated multi-line secret. + @pytest.mark.parametrize("hijack", ["CE_ARGS", "CE_RUN_DIR", "GITHUB_OUTPUT"]) + def test_a_newline_in_a_value_cannot_rewrite_the_step(self, run_script, tmp_path, hijack): + rc, argv, out = _coder_eval( + run_script, + tmp_path, + CE_ARGS="tasks/real.yaml\n", + CE_RUN_DIR="runs/ci", + CE_ENV=f"API_BASE=x\n{hijack}=/tmp/hijacked\n", + ) + assert rc == 0, out + assert "tasks/real.yaml" in argv, "the caller's task path was dropped" + assert "/tmp/hijacked" not in argv + assert argv[:5] == ["run", "--run-dir", "runs/ci", "--junit-xml", "runs/ci/junit.xml"] + + # PATH is the sharpest of these: it decides WHICH coder-eval runs, and the + # name filter admits it. $GITHUB_PATH is the scoped, log-visible alternative. + @pytest.mark.parametrize("name", ["PATH", "BASH_ENV", "LD_PRELOAD", "IFS"]) + def test_reserved_names_are_rejected_before_running(self, run_script, tmp_path, name): + rc, argv, out = _coder_eval(run_script, tmp_path, CE_ENV=f"{name}=/tmp/evil") + assert rc != 0 + assert argv == [], "coder-eval ran despite a reserved env name" + assert "reserved name" in out + assert name in out + + +class TestNoWorkflowExpressionsInScripts: + """A `run:` body must carry no `${{ ... }}`. + + Two reasons, one of which has already bitten. GitHub parses every `${{ }}` + inside a block scalar before bash ever sees it, so a malformed one -- even + inside a shell COMMENT -- fails the whole action template to load with + `An expression was expected` and a line number pointing at `run: |`, which + says nothing about where the text actually is. And a well-formed one would + be textually substituted before bash parses the line, which is the injection + pattern GitHub's own hardening guidance rejects. Every value this action + needs already arrives through the step's `env:` block. + """ + + @pytest.mark.parametrize("step_name", ["Install coder-eval", "Run coder-eval"]) + def test_no_expression_syntax_in_a_run_body(self, step_name): + script = _step_script(step_name) + assert "${{" not in script, ( + f"the {step_name!r} script contains '${{{{' -- GitHub evaluates it before bash, " + "so even a comment breaks the template. Pass the value through the step's env: block." + ) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 181a6734..64508fe7 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -1753,8 +1753,11 @@ def test_ci_skill_covers_experiments_and_pins(self): # drops the `agent:` config it supplies, so the gate measures something other than # what the suite measures locally; and a `version:` input that ignores the repo's # pin runs the gate on a different CLI than the repo is authored against. - text = " ".join((PLUGIN_ROOT / "skills" / "ci" / "SKILL.md").read_text(encoding="utf-8").split()) - assert "extra-args" in text and "experiment" in text, ( + raw = (PLUGIN_ROOT / "skills" / "ci" / "SKILL.md").read_text(encoding="utf-8") + text = " ".join(raw.split()) + # A standalone `-e` token, not the substring inside "coder-eval": the experiment + # now rides in `args`, one argument per line, so the flag stands on its own. + assert "-e" in raw.split() and "experiment" in text, ( "the ci skill does not say how to pass an experiment through to the run — a " "suite that resolves through one silently measures something else without it" ) @@ -1762,38 +1765,6 @@ def test_ci_skill_covers_experiments_and_pins(self): "the ci skill no longer conditions the `version:` input on whether the repository pins a coder-eval version" ) - def test_ci_skill_does_not_recommend_a_recursive_task_glob(self): - # `action.yml` expands the `tasks:` input unquoted (`args+=($CE_TASKS)`) with - # globstar OFF, so `a/**/*.yaml` degrades to `a/*/*.yaml` and silently drops every - # top-level task — a depth-dependent "measured the wrong set" bug. nullglob is off - # too, so an unmatched depth pattern reaches the CLI literally and exits 1. Both - # reproduced by hand. The snippet must therefore show neither `**` in its tasks - # value nor a fixed ladder of depths. - skill = PLUGIN_ROOT / "skills" / "ci" / "SKILL.md" - assert "globstar" in skill.read_text(encoding="utf-8"), ( - "the ci skill emits explicit globs but no longer says WHY — without the reason, " - "the next reader simplifies them back to `**` and loses the top-level tasks" - ) - - # Scoped to every surface CE026 already scans, not just the skill: the `ci` skill was - # taught to avoid `**` while five snippets across README.md, docs/CI_GATE.md and the - # CI tutorial still showed `tasks: tests/tasks/**/*.yaml`, so the plugin contradicted - # the repo's own onboarding docs — and those are the ones integrators copy. - from tests.lint.action_docs import default_doc_paths - - offenders = [ - f"{path}:{n}: {line.strip()}" - for path in default_doc_paths(Path(__file__).parent.parent) - for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1) - if re.search(r"^\s*tasks:.*\*\*", line) - ] - assert not offenders, ( - "recursive `**` glob in a documented `tasks:` input value — the action word-splits " - "and pathname-expands that value with globstar off, so it silently drops every task " - "above the deepest matching level. Emit explicit per-depth globs or a file " - "list:\n\n" + "\n".join(f" {o}" for o in offenders) - ) - def test_check_skill_detects_before_scaffolding(self): # Scaffolding a second activation suite beside one that already covers the same # skill is worse than doing nothing: two suites drift, and the user pays for both @@ -2292,7 +2263,7 @@ def test_action_input_names_reads_the_real_action(self): from tests.lint.action_docs import action_input_names names = action_input_names(self.ACTION_YML) - assert {"tasks", "junit-path", "env"} <= names, names + assert {"args", "run-dir", "env"} <= names, names def test_catches_an_unknown_action_input(self, tmp_path: Path): from tests.lint.action_docs import find_unknown_action_inputs diff --git a/tests/test_run_helpers.py b/tests/test_run_helpers.py new file mode 100644 index 00000000..cc43f35a --- /dev/null +++ b/tests/test_run_helpers.py @@ -0,0 +1,79 @@ +"""Task-path expansion, which decides what a CI gate actually measures. + +`expand_task_files` had no direct test: its three other references all patch it +out. The contract the docs and the `ci` skill publish is that a glob matching +nothing exits 1 -- so a stale entry in a multi-line `args:` block cannot leave a +gate green over tasks it never ran. +""" + +from pathlib import Path + +import pytest +import typer + +from coder_eval.cli.run_helpers import expand_task_files + + +@pytest.fixture +def tasks_tree(tmp_path, monkeypatch): + """A suite at two depths, so `**` recursion is exercised for real.""" + (tmp_path / "tasks" / "sub").mkdir(parents=True) + (tmp_path / "tasks" / "top.yaml").write_text("task_id: top\n", encoding="utf-8") + (tmp_path / "tasks" / "sub" / "deep.yaml").write_text("task_id: deep\n", encoding="utf-8") + (tmp_path / "empty").mkdir() + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _names(paths): + return sorted(p.name for p in paths) + + +class TestRecursiveGlob: + # The published snippets all use `**`, and the lint rule that used to ban it + # was removed in favour of this promise. A switch to `glob.glob`, which is + # non-recursive by default, would silently drop the top-level task. + def test_double_star_matches_both_depths(self, tasks_tree): + assert _names(expand_task_files([Path("tasks/**/*.yaml")])) == [ + "deep.yaml", + "top.yaml", + ] + + def test_a_literal_file_is_passed_through(self, tasks_tree): + + assert _names(expand_task_files([Path("tasks/top.yaml")])) == ["top.yaml"] + + +class TestFailsClosed: + def test_a_single_unmatched_pattern_exits(self, tasks_tree): + + with pytest.raises(typer.Exit): + expand_task_files([Path("nope/*.yaml")]) + + # The regression this guards: accumulating across patterns and checking only + # the union meant one renamed suite among several ran the survivors and + # exited 0, reporting green over unmeasured tasks. + def test_one_stale_pattern_among_several_exits(self, tasks_tree): + + with pytest.raises(typer.Exit): + expand_task_files([Path("tasks/*.yaml"), Path("renamed/*.yaml")]) + + def test_the_unmatched_pattern_is_named(self, tasks_tree, capsys): + + with pytest.raises(typer.Exit): + expand_task_files([Path("tasks/*.yaml"), Path("renamed/*.yaml")]) + out = capsys.readouterr().out + assert "renamed" in out + # The one that DID match is not reported as a failure. + assert "no match: tasks/" not in out + + # A directory that exists but holds no task file is the same failure as a + # typo: the caller named it and expected tasks there. + def test_an_empty_directory_exits(self, tasks_tree): + + with pytest.raises(typer.Exit): + expand_task_files([Path("empty/*.yaml")]) + + def test_no_patterns_at_all_exits(self, tasks_tree): + with pytest.raises(typer.Exit): + expand_task_files([])