diff --git a/.gitignore b/.gitignore index 2ad773561..adf0432ab 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,11 @@ local/ tests/plt_* tests/graders/skills/*.json +# Skill functional-test fixtures — tracked despite the *.jsonl rule above; +# only the fixtures themselves, not generated results/ (ignored below). +!skills/**/tests/*_test_cases.jsonl +skills/**/tests/results/ + # Security Files *.pem *.key diff --git a/cookbooks/skills_evaluation/skill_models.py b/cookbooks/skills_evaluation/skill_models.py index 792a6a2fa..e5e41da45 100644 --- a/cookbooks/skills_evaluation/skill_models.py +++ b/cookbooks/skills_evaluation/skill_models.py @@ -209,7 +209,7 @@ def _read_text(path: Path) -> str: class SkillLoader: """Loads Agent Skill packages from a directory. - Supports two directory layouts: + Supports individual skills, flat collections, and nested domain suites: **Single skill**:: @@ -225,6 +225,15 @@ class SkillLoader: scripts/review.py paper-review/ SKILL.md + + **Domain suites** (grouping directories may be nested):: + + skills_dir/ + academic-eval/ + 01-paper-review/ + SKILL.md + standalone/ + SKILL.md """ @classmethod @@ -330,7 +339,10 @@ def load_from_directory(cls, skills_dir: Union[str, Path]) -> List[SkillPackage] Args: skills_dir: Path to a directory. If the directory itself contains ``SKILL.md`` it is treated as a single-skill directory; otherwise - each immediate subdirectory is checked for a ``SKILL.md``. + subdirectories are searched recursively. Discovery stops at + any directory containing ``SKILL.md`` so bundled examples are + not treated as separate packages. Each resolved directory is + visited once, including when reached through a symbolic link. Returns: List of successfully loaded :class:`SkillPackage` objects (may be empty). @@ -342,20 +354,26 @@ def load_from_directory(cls, skills_dir: Union[str, Path]) -> List[SkillPackage] if not skills_dir.is_dir(): raise ValueError(f"Not a directory: {skills_dir}") - if (skills_dir / SKILL_MD_NAME).is_file(): - skill = cls.load_skill(skills_dir) - return [skill] if skill else [] - skills: List[SkillPackage] = [] - for subdir in sorted(skills_dir.iterdir()): - if not subdir.is_dir(): - continue - if any(p in _IGNORE_DIRS for p in subdir.parts): - continue - skill = cls.load_skill(subdir) - if skill: - skills.append(skill) + visited: set[Path] = set() + + def collect(directory: Path) -> None: + resolved = directory.resolve() + if resolved in visited: + return + visited.add(resolved) + + if (directory / SKILL_MD_NAME).is_file(): + skill = cls.load_skill(directory) + if skill: + skills.append(skill) + return + + for subdir in sorted(directory.iterdir()): + if subdir.is_dir() and subdir.name not in _IGNORE_DIRS: + collect(subdir) + collect(skills_dir) return skills diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 000000000..c3cb35851 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,112 @@ +# Skills + +This directory holds Agent Skills (Anthropic Agent Skill protocol — YAML +frontmatter + Markdown body) for OpenJudge. Every `SKILL.md`, wherever it +lives, is **independently installable**: it carries everything it needs +inline, so a single `/SKILL.md` folder can be pulled out and installed +on its own in Claude Code, Cursor, Codex, Hermes, OpenClaw, or any other +client that speaks the same protocol — you never need this README or a +suite's own README to make one skill work. + +Grouping below **is folder structure only** — there is no code-level registry +(no `SUITE_REGISTRY`, no `DomainSuite` class). A "domain suite" is just a +directory of related `/SKILL.md` workflows plus a `README.md` that +explains how they relate. + +## Domain suites (multi-skill, folder-grouped) + +| Suite | Skills | Dimension | What it covers | +|---|---|---|---| +| [`eval_pipeline/`](eval_pipeline/) | 9 (`00-meta-eval` … `08-bootstrap`) | Horizontal methodology | How to build an evaluation system from scratch — dataset design, metric/grader selection, human-alignment calibration, reporting, plus RAG/prompt-regression/redteam/bootstrap scenarios. Router at `00-meta-eval`. | +| [`academic-eval/`](academic-eval/) | 4 (`00-academic-router`, `01-paper-review`, `02-bib-verify`, `03-ref-hallucination-arena`) | Vertical domain | Academic paper review, BibTeX verification, and citation-hallucination benchmarking. | +| [`arena-eval/`](arena-eval/) | 3 (`00-arena-router`, `01-auto-arena`, `02-ref-hallucination-arena`) | Vertical domain | Arena-style model/agent comparison — generic win-rate ranking, and citation-hallucination-specific ranking. | +| [`openjudge-core/`](openjudge-core/) | 2 (`01-graders-and-pipeline`, `02-rl-reward`) | Horizontal methodology | Direct OpenJudge core-library API: graders/`GradingRunner`/aggregators, and RL reward-signal construction. No router — see the suite's own README for why. | + +`eval_pipeline` is the range-of-motion paradigm all the other suites above +were reshaped to match (three-layer structure, directory-as-grouping, router +only where sub-skills are genuinely ambiguous, every `/SKILL.md` +independently installable). See +[`docs/superpowers/specs/2026-07-09-skills-domain-suite-proposal.md`](../docs/superpowers/specs/2026-07-09-skills-domain-suite-proposal.md) +for the full design rationale. + +### Functional test coverage + +`eval_pipeline`, `academic-eval`, and `arena-eval` each have an actor+judge +functional test harness under their own `tests/` folder — a *runner* +(shared, parametrized: `eval_pipeline/tests/run_eval_pipeline_skill_tests.py +--skill-root --cases /tests/_test_cases.jsonl`), a +suite-specific JSONL fixture, and a testing guide. `openjudge-core` and the +isolated skills don't have this yet (see each suite's `README.md` under +"Validating the skills themselves" for the ones that do). + +### `ref-hallucination-arena` — intentional duplication, not a bug + +`academic-eval/03-ref-hallucination-arena/SKILL.md` and +`arena-eval/02-ref-hallucination-arena/SKILL.md` start as full copies of the +same content and are **independently maintained** — no shared `references/` +directory, no symlink, no sync script. This is a direct consequence of the +independent-installability constraint: a skill can't rely on content that +lives outside its own folder. The two copies are free to diverge (the +academic copy can lean into "how trustworthy are this paper's citations", +the arena copy into "how does this model compare to others"); duplicated +content is an accepted cost, not a problem to solve. + +## Isolated skills (single-skill, no suite directory) + +These have no other skill in this repo referencing them, so they stay at +`skills//SKILL.md` with no suite folder — the simplest possible +grouping (1 skill = 1 domain). + +| Skill | What it covers | +|---|---| +| [`claude-authenticity/`](claude-authenticity/) | Detect whether an API endpoint is genuinely backed by Claude (vs. a wrapper/proxy/impersonator); extract injected system prompts. Zero OpenJudge dependency. | +| [`mmx-cli/`](mmx-cli/) | Generate text/image/video/speech/music via the MiniMax AI platform. Third-party CLI wrapper, zero OpenJudge dependency. | +| [`find-skills-combo/`](find-skills-combo/) | Discover and recommend combinations of skills from the **external** open agent-skills ecosystem to cover a multi-part task. Unrelated to this repo's own suite grouping. | + +**Future rule**: if an isolated skill above ever becomes something a domain +suite depends on or wraps, move the whole folder into that suite's directory +(same copy-don't-link approach as `ref-hallucination-arena` above, or a +rename+move if a single owner is clear) rather than keeping it independent +and cross-linking from two places. + +## Breaking change: path migration (this reorg) + +This reorg moved 6 previously-published skills to new install paths. There +are no redirect stubs — installing the old path will 404; use the new path. + +| Old path | New path | +|---|---| +| `skills/paper-review/` | `skills/academic-eval/01-paper-review/` | +| `skills/bib-verify/` | `skills/academic-eval/02-bib-verify/` | +| `skills/ref-hallucination-arena/` | `skills/academic-eval/03-ref-hallucination-arena/` **and** `skills/arena-eval/02-ref-hallucination-arena/` (two independent copies — see above) | +| `skills/auto-arena/` | `skills/arena-eval/01-auto-arena/` | +| `skills/openjudge/` | `skills/openjudge-core/01-graders-and-pipeline/` | +| `skills/rl-reward/` | `skills/openjudge-core/02-rl-reward/` | + +The migrated skills' frontmatter `name` values also change to match their +new directory names. Update explicit skill invocations to use the final +component of each new path (for example, `paper-review` becomes +`01-paper-review`, and `openjudge` becomes `01-graders-and-pipeline`). The +two reference-arena copies use distinct names: `03-ref-hallucination-arena` +in `academic-eval` and `02-ref-hallucination-arena` in `arena-eval`. + +`SkillLoader.load_from_directory("skills")` discovers skills recursively +through suite directories. A directory containing `SKILL.md` is treated as +one package, including its bundled references and examples. + +`skills/eval_pipeline/`, `skills/claude-authenticity/`, `skills/mmx-cli/`, +and `skills/find-skills-combo/` are unaffected. + +## Adding a new skill + +- **Fits an existing suite's methodology or domain?** Add it as + `//SKILL.md` with `name: NN-name` matching its directory, + choose a name unique across the repository, update that suite's `README.md`, + and add a router entry only if it creates real selection ambiguity with a + sibling (see each suite's README for its router-inclusion rationale). +- **Genuinely stands alone?** Add it as `skills//SKILL.md` — no suite + folder needed. +- **Either way**, the skill's `SKILL.md` must be installable on its own: only + `name` + `description` in frontmatter, no links outside its own folder (or + its own suite folder), and any non-`openjudge`/`rl-reward` pip dependency + listed explicitly under Prerequisites. diff --git a/skills/academic-eval/00-academic-router/SKILL.md b/skills/academic-eval/00-academic-router/SKILL.md new file mode 100644 index 000000000..731d70461 --- /dev/null +++ b/skills/academic-eval/00-academic-router/SKILL.md @@ -0,0 +1,76 @@ +--- +name: 00-academic-router +description: > + Use when the user wants help with academic papers or citations but it's unclear + which specific workflow fits — reviewing a paper, checking a BibTeX file for fake + references, or benchmarking multiple LLMs on reference-recommendation accuracy. + Also use when the user mentions paper review, peer review, BibTeX verification, + citation checking, reference hallucination, or academic literature accuracy and + hasn't specified which of those three tasks they mean. This skill is the entry + router for the academic-eval suite: it asks one diagnostic question then routes + to the right sub-skill. +--- + +# Academic Eval Router + +Entry router for the `academic-eval` suite. You diagnose what the user actually +wants and route them to one of three sub-skills. You don't review papers, verify +BibTeX files, or run arena benchmarks yourself — you're the triage desk. + +Each sub-skill is self-contained: it carries inline everything it needs, so it can +be installed and used on its own. + +## Diagnostic Question + +Ask (unless the user's request already makes the answer obvious): + +``` +To route you correctly, which of these matches what you want? + +a) Review a single paper (PDF or LaTeX source) for correctness/quality/novelty + — optionally also check its bibliography +b) Check a standalone .bib file for fabricated or mismatched references + (no paper review needed) +c) Benchmark/compare multiple LLMs on how often they hallucinate references + when asked to recommend citations (arena-style, many queries) +``` + +**Shortcut rule**: if the user already said "review my paper", "check this PDF", +"verify this .bib file", or "compare models on reference hallucination", skip the +question — the routing is already clear from their phrasing. + +## Triage Table + +| User says / has | Use workflow | What it does | +|---|---|---| +| "Review this paper" (PDF or `.tar.gz`/`.zip` TeX source) | `01-paper-review` | Multi-stage review: safety, correctness, quality/novelty score, criticality — optionally + BibTeX check | +| "Review this paper AND check its references" | `01-paper-review` | Same pipeline with `--bib` set — one run covers both | +| "Just check this .bib file, no paper" | `02-bib-verify` | Cross-checks every entry against CrossRef/arXiv/DBLP, flags `verified`/`suspect`/`not_found` | +| "Compare N models on how often they cite fake papers" / "benchmark reference hallucination rate" | `03-ref-hallucination-arena` | Runs many recommendation queries per model, verifies every returned reference, ranks models by hallucination rate | +| "Compare models on general quality/response, not specifically citations" | — | Not this suite — see the `arena-eval` suite's `01-auto-arena` instead | + +## Key distinctions + +- **`01-paper-review` vs `02-bib-verify`**: both use the same underlying + `cookbooks.paper_review` pipeline. Use `01-paper-review` whenever a paper file + exists (even if the *only* thing the user cares about is the bibliography — + `--bib_only` mode is documented there). Use `02-bib-verify` only when there is + **no paper**, just a loose `.bib` file to sanity-check. +- **`01-paper-review`/`02-bib-verify` vs `03-ref-hallucination-arena`**: the first + two evaluate *one document's* existing references after the fact. The third + evaluates *model behavior* — how often a model invents fake citations when + asked to recommend some, across a benchmark of queries and models. If the user + wants a leaderboard/ranking of models, not a report on one document, route to + `03-ref-hallucination-arena`. + +## Output + +``` +Recommended workflow: `[skill-name]` + +Why: [one sentence tying the user's request to the triage table row] +``` + +Recommend exactly one workflow. If the request spans two (e.g., "review this +paper, and separately benchmark 3 models on citation accuracy"), say so +explicitly and give both, in the order the user would naturally do them. diff --git a/skills/paper-review/SKILL.md b/skills/academic-eval/01-paper-review/SKILL.md similarity index 99% rename from skills/paper-review/SKILL.md rename to skills/academic-eval/01-paper-review/SKILL.md index 98191f07d..dbf74b7ca 100644 --- a/skills/paper-review/SKILL.md +++ b/skills/academic-eval/01-paper-review/SKILL.md @@ -1,5 +1,5 @@ --- -name: paper-review +name: 01-paper-review description: > Review academic papers for correctness, quality, and novelty using OpenJudge's multi-stage pipeline. Supports PDF files and LaTeX source packages (.tar.gz/.zip). diff --git a/skills/paper-review/reference.md b/skills/academic-eval/01-paper-review/reference.md similarity index 100% rename from skills/paper-review/reference.md rename to skills/academic-eval/01-paper-review/reference.md diff --git a/skills/bib-verify/SKILL.md b/skills/academic-eval/02-bib-verify/SKILL.md similarity index 92% rename from skills/bib-verify/SKILL.md rename to skills/academic-eval/02-bib-verify/SKILL.md index d64576a6e..0603efde1 100644 --- a/skills/bib-verify/SKILL.md +++ b/skills/academic-eval/02-bib-verify/SKILL.md @@ -1,5 +1,5 @@ --- -name: bib-verify +name: 02-bib-verify description: > Verify a BibTeX file for hallucinated or fabricated references by cross-checking every entry against CrossRef, arXiv, and DBLP. Reports each reference as @@ -73,5 +73,5 @@ Each reference entry is assigned one of three statuses: ## Additional resources -- Full pipeline options: [../paper-review/reference.md](../paper-review/reference.md) -- Combined PDF review + BibTeX verification: [../paper-review/SKILL.md](../paper-review/SKILL.md) +- Full pipeline options: [../01-paper-review/reference.md](../01-paper-review/reference.md) +- Combined PDF review + BibTeX verification: [../01-paper-review/SKILL.md](../01-paper-review/SKILL.md) diff --git a/skills/ref-hallucination-arena/SKILL.md b/skills/academic-eval/03-ref-hallucination-arena/SKILL.md similarity index 98% rename from skills/ref-hallucination-arena/SKILL.md rename to skills/academic-eval/03-ref-hallucination-arena/SKILL.md index 6f768842b..18266eb9c 100644 --- a/skills/ref-hallucination-arena/SKILL.md +++ b/skills/academic-eval/03-ref-hallucination-arena/SKILL.md @@ -1,5 +1,5 @@ --- -name: ref-hallucination-arena +name: 03-ref-hallucination-arena description: > Benchmark LLM reference recommendation capabilities by verifying every cited paper against Crossref, PubMed, arXiv, and DBLP. Measures hallucination rate, @@ -254,7 +254,7 @@ evaluation_results/ref_hallucination_arena/ ## Additional resources -- Full config examples: [cookbooks/ref_hallucination_arena/examples/](../../cookbooks/ref_hallucination_arena/examples/) -- Documentation: [docs/validating_graders/ref_hallucination_arena.md](../../docs/validating_graders/ref_hallucination_arena.md) +- Full config examples: [cookbooks/ref_hallucination_arena/examples/](../../../cookbooks/ref_hallucination_arena/examples/) +- Documentation: [docs/validating_graders/ref_hallucination_arena.md](../../../docs/validating_graders/ref_hallucination_arena.md) - Official dataset: [HuggingFace](https://huggingface.co/datasets/OpenJudge/ref-hallucination-arena) - Leaderboard: [openjudge.me/leaderboard](https://openjudge.me/leaderboard) diff --git a/skills/academic-eval/README.md b/skills/academic-eval/README.md new file mode 100644 index 000000000..194f65188 --- /dev/null +++ b/skills/academic-eval/README.md @@ -0,0 +1,63 @@ +# Academic Eval — paper review & citation accuracy + +A set of skills for academic-paper workflows built on OpenJudge: reviewing a +paper end-to-end, spot-checking a bibliography for fabricated references, or +benchmarking how often LLMs hallucinate citations at scale. + +Each skill is a self-contained workflow in `/SKILL.md`. Start at +`00-academic-router` if you're not sure which one you need. + +## The workflows + +| # | Skill | Use it when | +|---|---|---| +| 00 | `00-academic-router` | You're not sure whether you want a paper review, a BibTeX check, or an arena-style benchmark. | +| 01 | `01-paper-review` | You have a paper (PDF or LaTeX source) and want a multi-stage review — safety, correctness, quality/novelty, criticality, optionally + BibTeX. | +| 02 | `02-bib-verify` | You have a standalone `.bib` file (no paper) and want to check it for fabricated/mismatched references. | +| 03 | `03-ref-hallucination-arena` | You want to benchmark/compare multiple LLMs on how often they invent fake citations, across many queries. | + +## Relationship between the workflows + +`01-paper-review` and `02-bib-verify` both run on top of the same +`cookbooks.paper_review` pipeline — `01` is the full document review (with an +optional `--bib` flag to also verify references), `02` is the BibTeX-only mode +for when there's no paper to review, just a bibliography to sanity-check. + +`03-ref-hallucination-arena` is a different axis entirely: instead of +evaluating one document's existing references, it evaluates *model behavior* — +how often a model fabricates references when asked to recommend citations, +scored across a benchmark of queries and ranked across models. It shares +`ref-hallucination-arena`'s content with the `arena-eval` suite's +`02-ref-hallucination-arena` — the two copies are independently maintained +(see the root [`skills/README.md`](../README.md) for why). + +## Dependencies + +| Skill | Cookbook | +|---|---| +| `01-paper-review` | `cookbooks/paper_review/` | +| `02-bib-verify` | `cookbooks/paper_review/` (BibTeX-only mode) | +| `03-ref-hallucination-arena` | `cookbooks/ref_hallucination_arena/` | + +```bash +pip install py-openjudge litellm +pip install matplotlib # only needed by 03-ref-hallucination-arena (charts) +``` + +## Self-contained skills + +Each `/SKILL.md` is self-contained per the Anthropic Agent Skill +protocol — it can be installed and used on its own without this README or the +rest of the suite. Cross-references between skills in this suite use relative +links scoped to this directory (e.g. `02-bib-verify` links to +`../01-paper-review/`); there are no links out to other suites. + +## Validating the skills themselves + +This suite has an actor+judge functional test harness, same pattern as +`eval_pipeline`: an *actor* model follows a skill to answer a realistic user +request, and a separate *judge* model grades the answer against the case's +acceptance criteria. See [`tests/academic_eval_testing_guide.md`](tests/academic_eval_testing_guide.md) +for how to run it — it reuses `eval_pipeline`'s runner (parametrized, not +duplicated) against this suite's own 12 test cases in +[`tests/academic_eval_test_cases.jsonl`](tests/academic_eval_test_cases.jsonl). diff --git a/skills/academic-eval/tests/academic_eval_skill_audit.md b/skills/academic-eval/tests/academic_eval_skill_audit.md new file mode 100644 index 000000000..f39d10925 --- /dev/null +++ b/skills/academic-eval/tests/academic_eval_skill_audit.md @@ -0,0 +1,73 @@ +# Academic Eval Skill Audit + +Initial functional-test audit of the `academic-eval` suite, run right after the +suite was created (folder-migrated from `paper-review`/`bib-verify`/ +`ref-hallucination-arena` + new `00-academic-router`). + +## Run 1 — full set, `--repeat 1` + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir skills/academic-eval/tests/results \ + --report-prefix academic_eval +``` + +Actor: `qwen3.6-plus` · Judge: `qwen3-max` + +**Result: 12/12 pass, 0 partial, 0 fail.** + +| Case | Skill | Verdict | Score | +|---|---|---:|---:| +| `academic_router_001_route_paper_review` | `00-academic-router` | pass | 1.00 | +| `academic_router_002_bib_only_no_paper` | `00-academic-router` | pass | 1.00 | +| `academic_router_003_paper_plus_bib` | `00-academic-router` | pass | 1.00 | +| `academic_router_004_arena_benchmark_request` | `00-academic-router` | pass | 1.00 | +| `academic_router_005_general_quality_not_citations` | `00-academic-router` | pass | 1.00 | +| `paper_review_001_model_fallback_no_explicit_model` | `01-paper-review` | pass | 1.00 | +| `paper_review_002_bad_request_base_url_diagnosis` | `01-paper-review` | pass | 1.00 | +| `bib_verify_001_standalone_bib_zh_report` | `02-bib-verify` | pass | 1.00 | +| `bib_verify_002_interpret_suspect_entry` | `02-bib-verify` | pass | 1.00 | +| `ref_arena_001_dataset_format_check` | `03-ref-hallucination-arena` | pass | 1.00 | +| `ref_arena_002_interpret_low_accuracy` | `03-ref-hallucination-arena` | pass | 1.00 | +| `academic_eval_001_end_to_end_route_and_execute` | `academic_eval_collection` | pass | 1.00 | + +## Run 2 — variance check, `--repeat 3` on the two most ambiguity-prone router cases + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir /tmp/academic_eval_repeat_check --report-prefix academic_eval \ + --case-id academic_router_003_paper_plus_bib \ + --case-id academic_router_005_general_quality_not_citations \ + --repeat 3 +``` + +- `academic_router_003_paper_plus_bib`: pass / pass / pass — score 1.00 (n=3) +- `academic_router_005_general_quality_not_citations`: pass / pass / pass — score 1.00 (n=3) + +No variance observed across 3 runs on either boundary case. + +## Reading + +The suite's 4 skills (router + 3 migrated originals) pass their functional +tests immediately after the folder migration and router rewrite — no +follow-up SKILL.md patch was needed. The router's triage-table wording for +the two trickiest rows (combined paper+bib request; out-of-scope redirect to +`arena-eval`) held up under repeat sampling. + +**Caveat**: 21/21 clean passes across two suites on a first run is a stronger +result than typical for a brand-new skill collection (`eval_pipeline`'s own +first audit — see `../../eval_pipeline/tests/eval_pipeline_skill_audit.md` — +needed several fix rounds). Two contributing factors, not just "this suite +is more polished": (1) most of the SKILL.md content is unchanged domain +documentation carried over from the pre-migration skills, which had already +been used in practice; (2) all 12 test cases here were authored by the same +person who wrote the router content in the same sitting, so criteria and +skill text may share blind spots that an independently-written test set or a +different actor/judge model pairing would catch. Re-running with `--repeat +5`, a different judge model, or independently authored adversarial cases is +the natural next step before treating this as a strong reliability claim +rather than an initial smoke signal. diff --git a/skills/academic-eval/tests/academic_eval_test_cases.jsonl b/skills/academic-eval/tests/academic_eval_test_cases.jsonl new file mode 100644 index 000000000..5e6f902f4 --- /dev/null +++ b/skills/academic-eval/tests/academic_eval_test_cases.jsonl @@ -0,0 +1,12 @@ +{"id": "academic_router_001_route_paper_review", "skill": "00-academic-router", "category": "routing", "user_request": "I have a PDF of my NeurIPS submission and want a full review — check correctness, novelty, and quality before I submit.", "artifacts": {}, "expected_behavior": ["Recognize this as a full paper review request, not a bib-only or arena request.", "Recommend `01-paper-review` as the workflow.", "Not ask unnecessary clarifying questions when the request is already unambiguous."], "acceptance_criteria": ["Output explicitly recommends `01-paper-review`.", "Output does not recommend `02-bib-verify` or `03-ref-hallucination-arena` as the primary next step.", "Recommends exactly one workflow."], "failure_signals": ["Recommends `02-bib-verify` for a full paper review request.", "Recommends more than one workflow as the immediate next step.", "Asks the diagnostic question when the request already specifies a full review of a PDF."]} +{"id": "academic_router_002_bib_only_no_paper", "skill": "00-academic-router", "category": "routing", "user_request": "I don't have a paper to review, just this references.bib file. Can you tell me if any of these citations look fake?", "artifacts": {}, "expected_behavior": ["Recognize there is no paper, only a bibliography.", "Recommend `02-bib-verify`, not `01-paper-review`."], "acceptance_criteria": ["Output recommends `02-bib-verify`.", "Output explains that `02-bib-verify` is for a standalone .bib file with no paper."], "failure_signals": ["Recommends `01-paper-review` when the user explicitly said they have no paper.", "Recommends `03-ref-hallucination-arena` for checking one existing file."]} +{"id": "academic_router_003_paper_plus_bib", "skill": "00-academic-router", "category": "routing", "user_request": "Review my paper.pdf and also verify the references in refs.bib are real — one pass please.", "artifacts": {}, "expected_behavior": ["Recognize both a paper and its bibliography need checking in one pass.", "Recommend `01-paper-review` with its --bib option, not two separate workflows."], "acceptance_criteria": ["Output recommends `01-paper-review`, mentioning it can also verify the bibliography in the same run.", "Output does not tell the user to separately run `02-bib-verify` after `01-paper-review`."], "failure_signals": ["Recommends running `01-paper-review` and `02-bib-verify` as two separate steps for this request.", "Recommends `02-bib-verify` alone, ignoring the paper review request."]} +{"id": "academic_router_004_arena_benchmark_request", "skill": "00-academic-router", "category": "routing", "user_request": "We want to benchmark 4 different LLMs on how often they recommend fake papers when asked for citations, across about 100 test queries, and get a ranking.", "artifacts": {}, "expected_behavior": ["Recognize this is a multi-model benchmark, not a single-document check.", "Recommend `03-ref-hallucination-arena`."], "acceptance_criteria": ["Output recommends `03-ref-hallucination-arena`.", "Output does not recommend `01-paper-review` or `02-bib-verify`, which operate on one document, not a multi-model benchmark."], "failure_signals": ["Recommends `01-paper-review` or `02-bib-verify` for a multi-model ranking request.", "Fails to mention that a dataset of queries is needed."]} +{"id": "academic_router_005_general_quality_not_citations", "skill": "00-academic-router", "category": "routing", "user_request": "I want to compare 3 chatbot models on general response quality and helpfulness — not really about citations or papers.", "artifacts": {}, "expected_behavior": ["Recognize this request is outside academic-eval's scope (no papers or citations involved).", "Redirect the user to the arena-eval suite's `01-auto-arena` skill instead of forcing it into this suite."], "acceptance_criteria": ["Output states this isn't the right suite for a generic quality comparison.", "Output points to arena-eval's `01-auto-arena` as the correct skill."], "failure_signals": ["Tries to force the request into `01-paper-review`, `02-bib-verify`, or `03-ref-hallucination-arena`.", "Silently ignores that the request has nothing to do with papers or citations."]} +{"id": "paper_review_001_model_fallback_no_explicit_model", "skill": "01-paper-review", "category": "model_selection", "user_request": "Please review paper.pdf. I haven't told you which model to use.", "artifacts": {"available_api_keys": ["DASHSCOPE_API_KEY"]}, "expected_behavior": ["Since no model was specified and only DASHSCOPE_API_KEY is available, follow the fallback rule and choose a DashScope vision-capable model.", "Use the dashscope/ prefix convention."], "acceptance_criteria": ["Recommends `dashscope/qwen-vl-plus` (or another explicitly named DashScope vision model) as the model.", "Does not ask the user to also supply an OpenAI or Anthropic key when DashScope is already available.", "Mentions the model must support vision since PDF review defaults to vision mode."], "failure_signals": ["Recommends a non-vision-capable or text-only model for the default vision-mode PDF review.", "Recommends an OpenAI/Anthropic model despite no matching API key being available."]} +{"id": "paper_review_002_bad_request_base_url_diagnosis", "skill": "01-paper-review", "category": "troubleshooting", "user_request": "The paper review pipeline just failed with: 'BadRequestError: 400 - invalid request'. My --base_url is https://my-proxy.example.com/v1/chat/completions. What's wrong and how do I fix it?", "artifacts": {}, "expected_behavior": ["Diagnose the root cause: base_url ends with /v1/chat/completions instead of /v1, and litellm appends the path automatically.", "Give the concrete fix: strip everything after /v1 from --base_url.", "Not suggest bypassing the pipeline (e.g. reading the PDF as plain text and calling the API manually)."], "acceptance_criteria": ["Identifies the /v1/chat/completions suffix as the specific problem.", "Gives the corrected --base_url value ending in /v1.", "States that the full pipeline should be re-run after the fix, not bypassed."], "failure_signals": ["Recommends reading the PDF as text and manually calling the API instead of fixing and re-running the pipeline.", "Does not pinpoint the /v1/chat/completions vs /v1 issue.", "Just summarizes the paper itself as a workaround."]} +{"id": "bib_verify_001_standalone_bib_zh_report", "skill": "02-bib-verify", "category": "usage", "user_request": "I only have references.bib, no paper. Check it for fake citations and give me the report in Chinese, with my email your@email.com for better CrossRef rate limits.", "artifacts": {}, "expected_behavior": ["Use the bib-only mode of the paper_review pipeline.", "Include --bib_only, --email, and --language zh flags."], "acceptance_criteria": ["Command includes --bib_only references.bib (or equivalent explicit bib-only invocation).", "Command includes --email your@email.com.", "Command includes --language zh (or explicitly states Chinese report language).", "Explains the verified/suspect/not_found status meaning."], "failure_signals": ["Suggests running the full paper review pipeline on a nonexistent paper file.", "Omits the --email flag despite the user providing an email.", "Defaults to English report language despite the explicit request for Chinese."]} +{"id": "bib_verify_002_interpret_suspect_entry", "skill": "02-bib-verify", "category": "interpretation", "user_request": "One entry in my verification report came back as 'suspect' with title_match=false and author_match=true. What does that mean and should I be worried?", "artifacts": {"entry": {"status": "suspect", "title_match": false, "author_match": true, "year_match": true, "doi_match": false}}, "expected_behavior": ["Explain 'suspect' means title or authors don't match any real paper, i.e. likely mis-cited or fabricated.", "Note specifically that the title does not match even though authors/year do — flag as needing manual check, not dismiss it."], "acceptance_criteria": ["States that suspect means a manual check is recommended.", "Specifically calls out that title_match=false is the concerning field here.", "Does not classify this as verified."], "failure_signals": ["Claims the entry is fine/verified because author_match and year_match are true.", "Fails to mention the title mismatch as the key concern."]} +{"id": "ref_arena_001_dataset_format_check", "skill": "03-ref-hallucination-arena", "category": "dataset_design", "user_request": "I have 200 questions asking for paper recommendations, but they're just plain text strings in a .txt file, one per line. Can I use this directly with the arena benchmark?", "artifacts": {}, "expected_behavior": ["Explain the required dataset format: JSON/JSONL with a query field, plus optional discipline, num_refs, language, year_constraint.", "State that a plain .txt file must first be converted to the JSON/JSONL schema."], "acceptance_criteria": ["Mentions the dataset must be JSON or JSONL, not plain .txt.", "Lists at least the query field as required in each record.", "Does not claim the .txt file can be used as-is."], "failure_signals": ["Claims the .txt file works directly without conversion.", "Omits the required dataset schema fields."]} +{"id": "ref_arena_002_interpret_low_accuracy", "skill": "03-ref-hallucination-arena", "category": "interpretation", "user_request": "Model X came back with an overall verification rate (accuracy) of 45%. How good or bad is that?", "artifacts": {"overall_accuracy": 0.45}, "expected_behavior": ["Map 45% to the 'Fair: significant hallucination, use with caution' band (40-60%), not 'Good' or 'Excellent'."], "acceptance_criteria": ["Classifies 45% as 'Fair' (the 40-60% band), not Good/Excellent/Poor.", "States the model has significant hallucination and should be used with caution."], "failure_signals": ["Classifies 45% as 'Good' or 'Excellent'.", "Gives no interpretation at all, just repeats the number."]} +{"id": "academic_eval_001_end_to_end_route_and_execute", "skill": "academic_eval_collection", "category": "end_to_end", "user_request": "A colleague asks: 'Can you review my ICML draft and afterward tell me how it stacks up against 2 other LLMs at recommending real references for the same topic?' Walk through which skill(s) in this suite you'd use, in what order.", "artifacts": {}, "expected_behavior": ["Recognize this spans two separate needs: a document review, and a multi-model benchmark on reference recommendation.", "Sequence: first 01-paper-review for the draft, then 03-ref-hallucination-arena for the multi-model comparison — these are not the same workflow."], "acceptance_criteria": ["Names 01-paper-review for the draft review step.", "Names 03-ref-hallucination-arena for the multi-model reference-recommendation comparison.", "Does not conflate the two into a single workflow."], "failure_signals": ["Suggests only one skill covers both needs.", "Confuses the arena comparison with 02-bib-verify, which checks one document's existing references, not multiple models' recommendation behavior."]} diff --git a/skills/academic-eval/tests/academic_eval_test_cases.md b/skills/academic-eval/tests/academic_eval_test_cases.md new file mode 100644 index 000000000..ce15702dc --- /dev/null +++ b/skills/academic-eval/tests/academic_eval_test_cases.md @@ -0,0 +1,23 @@ +# Academic Eval Functional Test Cases (Summary) + +Source fixture: [`academic_eval_test_cases.jsonl`](academic_eval_test_cases.jsonl) +(the runner reads only the JSONL; this table is for human review). 12 cases +across the suite's 4 skills. + +| ID | Skill | Category | Request (abridged) | Key acceptance criterion | +|---|---|---|---|---| +| `academic_router_001_route_paper_review` | `00-academic-router` | routing | Full PDF review request | Routes to `01-paper-review` only | +| `academic_router_002_bib_only_no_paper` | `00-academic-router` | routing | Only a `.bib`, no paper | Routes to `02-bib-verify`, not `01-paper-review` | +| `academic_router_003_paper_plus_bib` | `00-academic-router` | routing | Review paper + verify its refs | One `01-paper-review` run with `--bib`, not two workflows | +| `academic_router_004_arena_benchmark_request` | `00-academic-router` | routing | Benchmark 4 LLMs on citation fabrication | Routes to `03-ref-hallucination-arena` | +| `academic_router_005_general_quality_not_citations` | `00-academic-router` | routing | Generic chatbot quality comparison | Redirects to `arena-eval`'s `01-auto-arena` | +| `paper_review_001_model_fallback_no_explicit_model` | `01-paper-review` | model_selection | No model given, only DashScope key | Picks `dashscope/qwen-vl-plus` (vision-capable) | +| `paper_review_002_bad_request_base_url_diagnosis` | `01-paper-review` | troubleshooting | 400 error, bad `--base_url` suffix | Diagnoses `/v1/chat/completions` vs `/v1`, re-runs pipeline | +| `bib_verify_001_standalone_bib_zh_report` | `02-bib-verify` | usage | Bib-only check, Chinese report + email | Correct `--bib_only`/`--email`/`--language zh` flags | +| `bib_verify_002_interpret_suspect_entry` | `02-bib-verify` | interpretation | `suspect` entry, title mismatch | Flags for manual check, not verified | +| `ref_arena_001_dataset_format_check` | `03-ref-hallucination-arena` | dataset_design | Plain `.txt` queries | Requires JSON/JSONL with `query` field | +| `ref_arena_002_interpret_low_accuracy` | `03-ref-hallucination-arena` | interpretation | 45% verification rate | Classifies as "Fair", not "Good" | +| `academic_eval_001_end_to_end_route_and_execute` | `academic_eval_collection` | end_to_end | Review + multi-model citation comparison | Names both `01-paper-review` and `03-ref-hallucination-arena` | + +See [`academic_eval_testing_guide.md`](academic_eval_testing_guide.md) for how +to run these and interpret results. diff --git a/skills/academic-eval/tests/academic_eval_testing_guide.md b/skills/academic-eval/tests/academic_eval_testing_guide.md new file mode 100644 index 000000000..f3a00707a --- /dev/null +++ b/skills/academic-eval/tests/academic_eval_testing_guide.md @@ -0,0 +1,137 @@ +# Academic Eval Functional Testing Guide + +This guide explains how to prove the `academic-eval` skills work on realistic +scenarios. It follows the same actor+judge functional-test pattern as +[`skills/eval_pipeline/tests/`](../../eval_pipeline/tests/eval_pipeline_testing_guide.md) — +this suite reuses that suite's runner rather than shipping a duplicate one +(see [Why a shared runner](#why-a-shared-runner) below). + +## What To Test + +1. Skill package quality test + - Use `cookbooks/skills_evaluation/evaluate_skills.py`. + - Grades whether the SKILL.md files are complete, relevant, safe, and well-designed. +2. Functional scenario test (this guide) + - Use `skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py` pointed at + `skills/academic-eval`. + - Checks whether a skill actually guides an agent to the right recommendation + (routing) or the right concrete answer (usage/troubleshooting/interpretation) + for a realistic user request. + +## API Key + +Same as `eval_pipeline`: two model roles, actor (follows the skill) and judge +(grades against acceptance criteria), defaulting to different models so one +model doesn't grade its own output. + +```bash +# .env at repo root — Aliyun DashScope (Bailian) +DASHSCOPE_API_KEY=sk-... + +# or, any OpenAI-compatible provider +export OPENAI_API_KEY="sk-..." +export OPENAI_BASE_URL="https://your-provider.example/v1" # omit for OpenAI itself +``` + +## Quick Smoke Test + +Run one router case first: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir skills/academic-eval/tests/results \ + --report-prefix academic_eval \ + --case-id academic_router_001_route_paper_review +``` + +Run a representative smoke set (one case per skill + the end-to-end case): + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir skills/academic-eval/tests/results \ + --report-prefix academic_eval \ + --case-id academic_router_002_bib_only_no_paper \ + --case-id paper_review_002_bad_request_base_url_diagnosis \ + --case-id bib_verify_002_interpret_suspect_entry \ + --case-id ref_arena_002_interpret_low_accuracy \ + --case-id academic_eval_001_end_to_end_route_and_execute +``` + +Run all cases: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir skills/academic-eval/tests/results \ + --report-prefix academic_eval +``` + +Outputs: + +```text +skills/academic-eval/tests/results/academic_eval_functional_report.md +skills/academic-eval/tests/results/academic_eval_functional_results.json +``` + +(`results/` is git-ignored — it regenerates on every run; the Markdown report +is for local/PR-description review, not something committed.) + +## Recommended Test Samples + +| Purpose | Case ID | Why | +|---|---|---| +| Router: paper vs bib-only | `academic_router_002_bib_only_no_paper` | Ensures a bib-only request doesn't get routed to the full review pipeline. | +| Router: combined request | `academic_router_003_paper_plus_bib` | Ensures "review + verify refs" collapses into one `01-paper-review` run, not two workflows. | +| Router: out-of-scope redirect | `academic_router_005_general_quality_not_citations` | Ensures the router doesn't force an unrelated request into this suite. | +| Paper review: troubleshooting | `paper_review_002_bad_request_base_url_diagnosis` | Ensures the skill diagnoses the `/v1/chat/completions` vs `/v1` bug instead of bypassing the pipeline. | +| Bib verify: interpretation | `bib_verify_002_interpret_suspect_entry` | Ensures a `suspect` entry isn't waved through as verified. | +| Ref arena: interpretation | `ref_arena_002_interpret_low_accuracy` | Ensures 45% accuracy is graded "Fair", not "Good". | +| End-to-end | `academic_eval_001_end_to_end_route_and_execute` | Ensures a request spanning two workflows isn't collapsed into one. | + +## Handling Variance + +Both actor and judge are LLMs — a single run is noisy. Use `--repeat 3` before +claiming a skill change worked: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/academic-eval \ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \ + --out-dir skills/academic-eval/tests/results \ + --report-prefix academic_eval \ + --case-id academic_router_003_paper_plus_bib --repeat 3 +``` + +## How To Interpret Results + +- `pass`: the skill gave enough guidance to satisfy all critical acceptance criteria. +- `partial`: the skill mostly worked but missed an artifact, caveat, or field. +- `fail`: the skill routed incorrectly or gave a misleading answer. + +Useful pass threshold for a first audit: smoke set 5/5 pass or partial with at +least 4 pass; full set at least 80% pass. + +## What To Do After A Failure + +1. Read the actor output in the Markdown report. +2. Compare it to the missed acceptance criteria. +3. Patch the relevant `SKILL.md` (or the router's triage table) with a clearer + rule or example. +4. Re-run only that case, then the smoke set. + +## Why a shared runner + +`skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py` is suite-agnostic +via `--skill-root` / `--cases` / `--out-dir` / `--report-prefix` — it derives +its own file paths from `__file__`, not from being inside a specific suite. +Copying the ~400-line runner into every suite's `tests/` folder would triple +the actor/judge prompt logic to keep in sync for no benefit: the runner is +internal test tooling, not something bound by the Agent Skill protocol's +independent-installability rule (that rule applies to `SKILL.md` content, not +to how the maintainers verify it). Only the JSONL fixture and this guide are +suite-specific. diff --git a/skills/arena-eval/00-arena-router/SKILL.md b/skills/arena-eval/00-arena-router/SKILL.md new file mode 100644 index 000000000..5ce684efc --- /dev/null +++ b/skills/arena-eval/00-arena-router/SKILL.md @@ -0,0 +1,83 @@ +--- +name: 00-arena-router +description: > + Use when the user wants to compare or benchmark multiple LLMs/agents + arena-style but it's unclear which specific workflow fits — a general-purpose + win-rate comparison on a custom task, or a benchmark specifically about + reference/citation hallucination rate. Also use when the user mentions model + arena, agent arena, pairwise model comparison, win-rate ranking, or comparing + models on a task and hasn't specified whether that task is generic or about + citation accuracy. This skill is the entry router for the arena-eval suite: + it asks one diagnostic question when needed, then recommends the workflow + or workflows needed to cover the request. +--- + +# Arena Eval Router + +Entry router for the `arena-eval` suite. You diagnose what the user wants to +compare models on and route them to the appropriate sub-skill or both when +the request spans both evaluation goals. You don't run comparisons yourself +— you're the triage desk. + +Each sub-skill is self-contained: it carries inline everything it needs, so it +can be installed and used on its own. + +## Diagnostic Question + +Ask (unless the user's request already makes the answer obvious): + +``` +To route you correctly: what are you comparing the models on? + +a) A custom task of your own choosing (chatbot quality, summarization, + coding, anything) — you'll get win-rate rankings from a judge model +b) Specifically how often each model fabricates or hallucinates references + when asked to recommend citations +``` + +**Shortcut rule**: if the user already said "run an arena eval on my chatbot +task" or "benchmark reference hallucination across these models", skip the +question — the routing is already clear from their phrasing. +Also skip the question when they explicitly ask for both general quality +and citation accuracy; recommend both workflows. + +## Triage Table + +| User says / has | Use workflow | What it does | +|---|---|---| +| "Compare/benchmark/rank these models on [any custom task]" | `01-auto-arena` | Generates queries from a task description, collects responses, auto-generates rubrics, runs pairwise judge comparisons, produces win-rate rankings | +| "Which model hallucinates citations least?" / "benchmark reference recommendation accuracy" | `02-ref-hallucination-arena` | Runs reference-recommendation queries per model, verifies every returned citation against CrossRef/PubMed/arXiv/DBLP, ranks by verified accuracy | +| "Compare general helpfulness AND citation accuracy" | `01-auto-arena`, then `02-ref-hallucination-arena` | Runs separate evaluations for judge preference and verified citation accuracy, preserving both goals | +| "I want to review one paper's existing bibliography, not compare models" | — | Not this suite — see the `academic-eval` suite's `01-paper-review` / `02-bib-verify` instead | + +## Key distinction + +Both workflows produce model rankings from head-to-head-style evaluation, but +differ in what "correct" means: + +- **`01-auto-arena`**: correctness is *judge opinion* — an LLM judge scores + pairwise which response is better for an arbitrary task. Works for any task, + needs no ground truth. +- **`02-ref-hallucination-arena`**: correctness is *externally verifiable* — + every cited reference is checked against real bibliographic databases + (CrossRef/PubMed/arXiv/DBLP), so the ranking reflects factual accuracy, not + judge preference. Narrower scope (citation recommendation only) but higher + ground-truth confidence. + +If the user cares only about citation accuracy, prefer +`02-ref-hallucination-arena` over `01-auto-arena` even if they phrase it as +"which model is better." + +## Output + +``` +Recommended workflow: `[skill-name]` + +Why: [one sentence tying the user's request to the triage table row] +``` + +Recommend one workflow when it covers the request. If the user asks for both +general quality and citation accuracy, recommend `01-auto-arena` followed by +`02-ref-hallucination-arena` as separate runs (or follow the user's requested +order). Explain that the two runs measure different things and report their +results separately; neither ranking substitutes for the other. diff --git a/skills/auto-arena/SKILL.md b/skills/arena-eval/01-auto-arena/SKILL.md similarity index 98% rename from skills/auto-arena/SKILL.md rename to skills/arena-eval/01-auto-arena/SKILL.md index e1cd8e75a..804443992 100644 --- a/skills/auto-arena/SKILL.md +++ b/skills/arena-eval/01-auto-arena/SKILL.md @@ -1,5 +1,5 @@ --- -name: auto-arena +name: 01-auto-arena description: > Automatically evaluate and compare multiple AI models or agents without pre-existing test data. Generates test queries from a task description, @@ -270,5 +270,5 @@ evaluation_results/ ## Additional resources -- Full config examples: [cookbooks/auto_arena/examples/](../../cookbooks/auto_arena/examples/) +- Full config examples: [cookbooks/auto_arena/examples/](../../../cookbooks/auto_arena/examples/) - Documentation: [Auto Arena Guide](https://agentscope-ai.github.io/OpenJudge/applications/auto_arena/) diff --git a/skills/arena-eval/02-ref-hallucination-arena/SKILL.md b/skills/arena-eval/02-ref-hallucination-arena/SKILL.md new file mode 100644 index 000000000..d43bce994 --- /dev/null +++ b/skills/arena-eval/02-ref-hallucination-arena/SKILL.md @@ -0,0 +1,260 @@ +--- +name: 02-ref-hallucination-arena +description: > + Benchmark LLM reference recommendation capabilities by verifying every cited + paper against Crossref, PubMed, arXiv, and DBLP. Measures hallucination rate, + per-field accuracy (title/author/year/DOI), discipline breakdown, and year + constraint compliance. Supports tool-augmented (ReAct + web search) mode. + Use when the user asks to evaluate, benchmark, or compare models on academic + reference hallucination, literature recommendation quality, or citation accuracy. +--- + +# Reference Hallucination Arena Skill + +Evaluate how accurately LLMs recommend real academic references using the +OpenJudge `RefArenaPipeline`: + +1. **Load queries** — from JSON/JSONL dataset +2. **Collect responses** — BibTeX-formatted references from target models +3. **Extract references** — parse BibTeX entries from model output +4. **Verify references** — cross-check against Crossref / PubMed / arXiv / DBLP +5. **Score & rank** — compute verification rate, per-field accuracy, discipline breakdown +6. **Generate report** — Markdown report + visualization charts + +## Prerequisites + +```bash +# Install OpenJudge +pip install py-openjudge + +# Extra dependency for ref_hallucination_arena (chart generation) +pip install matplotlib +``` + +## Gather from user before running + +| Info | Required? | Notes | +|------|-----------|-------| +| Config YAML path | Yes | Defines endpoints, dataset, verification settings | +| Dataset path | Yes | JSON/JSONL file with queries (can be set in config) | +| API keys | Yes | Env vars: `OPENAI_API_KEY`, `DASHSCOPE_API_KEY`, etc. | +| CrossRef email | No | Improves API rate limits for verification | +| PubMed API key | No | Improves PubMed rate limits | +| Output directory | No | Default: `./evaluation_results/ref_hallucination_arena` | +| Report language | No | `"en"` (default) or `"zh"` | +| Tavily API key | No | Required only if using tool-augmented mode | + +## Quick start + +### CLI + +```bash +# Run evaluation with config file +python -m cookbooks.ref_hallucination_arena --config config.yaml --save + +# Resume from checkpoint (default behavior) +python -m cookbooks.ref_hallucination_arena --config config.yaml --save + +# Start fresh, ignore checkpoint +python -m cookbooks.ref_hallucination_arena --config config.yaml --fresh --save + +# Override output directory +python -m cookbooks.ref_hallucination_arena --config config.yaml \ + --output_dir ./my_results --save +``` + +### Python API + +```python +import asyncio +from cookbooks.ref_hallucination_arena.pipeline import RefArenaPipeline + +async def main(): + pipeline = RefArenaPipeline.from_config("config.yaml") + result = await pipeline.evaluate() + + for rank, (model, score) in enumerate(result.rankings, 1): + print(f"{rank}. {model}: {score:.1%}") + +asyncio.run(main()) +``` + +## CLI options + +| Flag | Default | Description | +|------|---------|-------------| +| `--config` | — | Path to YAML configuration file (required) | +| `--output_dir` | config value | Override output directory | +| `--save` | `False` | Save results to file | +| `--fresh` | `False` | Start fresh, ignore checkpoint | + +## Minimal config file + +```yaml +task: + description: "Evaluate LLM reference recommendation capabilities" + +dataset: + path: "./data/queries.json" + +target_endpoints: + model_a: + base_url: "https://api.openai.com/v1" + api_key: "${OPENAI_API_KEY}" + model: "gpt-4" + system_prompt: "You are an academic literature recommendation expert. Recommend {num_refs} real papers in BibTeX format. Only recommend papers you are confident actually exist." + + model_b: + base_url: "https://dashscope.aliyuncs.com/compatible-mode/v1" + api_key: "${DASHSCOPE_API_KEY}" + model: "qwen3-max" + system_prompt: "You are an academic literature recommendation expert. Recommend {num_refs} real papers in BibTeX format. Only recommend papers you are confident actually exist." +``` + +## Full config reference + +### task + +| Field | Required | Description | +|-------|----------|-------------| +| `description` | Yes | Evaluation task description | +| `scenario` | No | Usage scenario | + +### dataset + +| Field | Default | Description | +|-------|---------|-------------| +| `path` | — | Path to JSON/JSONL dataset file (required) | +| `shuffle` | `false` | Shuffle queries before evaluation | +| `max_queries` | `null` | Max queries to use (`null` = all) | + +### target_endpoints.\ + +| Field | Default | Description | +|-------|---------|-------------| +| `base_url` | — | API base URL (required) | +| `api_key` | — | API key, supports `${ENV_VAR}` (required) | +| `model` | — | Model name (required) | +| `system_prompt` | built-in | System prompt; use `{num_refs}` placeholder | +| `max_concurrency` | `5` | Max concurrent requests for this endpoint | +| `extra_params` | — | Extra API request params (e.g. `temperature`) | +| `tool_config.enabled` | `false` | Enable ReAct agent with Tavily web search | +| `tool_config.tavily_api_key` | env var | Tavily API key | +| `tool_config.max_iterations` | `10` | Max ReAct iterations (1–30) | +| `tool_config.search_depth` | `"advanced"` | `"basic"` or `"advanced"` | + +### verification + +| Field | Default | Description | +|-------|---------|-------------| +| `crossref_mailto` | — | Email for Crossref polite pool | +| `pubmed_api_key` | — | PubMed API key | +| `max_workers` | `10` | Concurrent verification threads (1–50) | +| `timeout` | `30` | Per-request timeout in seconds | +| `verified_threshold` | `0.7` | Min composite score to count as VERIFIED | + +### evaluation + +| Field | Default | Description | +|-------|---------|-------------| +| `timeout` | `120` | Model API request timeout in seconds | +| `retry_times` | `3` | Number of retry attempts | + +### output + +| Field | Default | Description | +|-------|---------|-------------| +| `output_dir` | `./evaluation_results/ref_hallucination_arena` | Output directory | +| `save_queries` | `true` | Save loaded queries | +| `save_responses` | `true` | Save model responses | +| `save_details` | `true` | Save verification details | + +### report + +| Field | Default | Description | +|-------|---------|-------------| +| `enabled` | `true` | Enable report generation | +| `language` | `"zh"` | Report language: `"zh"` or `"en"` | +| `include_examples` | `3` | Examples per section (1–10) | +| `chart.enabled` | `true` | Generate charts | +| `chart.orientation` | `"vertical"` | `"horizontal"` or `"vertical"` | +| `chart.show_values` | `true` | Show values on bars | +| `chart.highlight_best` | `true` | Highlight best model | + +## Dataset format + +Each query in the JSON/JSONL dataset: + +```json +{ + "query": "Please recommend papers on Transformer architectures for NLP.", + "discipline": "computer_science", + "num_refs": 5, + "language": "en", + "year_constraint": {"min_year": 2020} +} +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `query` | Yes | Prompt for reference recommendation | +| `discipline` | No | `computer_science`, `biomedical`, `physics`, `chemistry`, `social_science`, `interdisciplinary`, `other` | +| `num_refs` | No | Expected number of references (default: 5) | +| `language` | No | `"zh"` or `"en"` (default: `"zh"`) | +| `year_constraint` | No | `{"exact": 2023}`, `{"min_year": 2020}`, `{"max_year": 2015}`, or `{"min_year": 2020, "max_year": 2024}` | + +Official dataset: [OpenJudge/ref-hallucination-arena](https://huggingface.co/datasets/OpenJudge/ref-hallucination-arena) + +## Interpreting results + +**Overall accuracy (verification rate):** +- **> 75%** — Excellent: model rarely hallucinates references +- **60–75%** — Good: most references are real, some fabrication +- **40–60%** — Fair: significant hallucination, use with caution +- **< 40%** — Poor: model frequently fabricates references + +**Per-field accuracy:** +- `title_accuracy` — % of titles matching real papers +- `author_accuracy` — % of correct author lists +- `year_accuracy` — % of correct publication years +- `doi_accuracy` — % of valid DOIs + +**Verification status:** +- `VERIFIED` — title + author + year all exactly match a real paper +- `SUSPECT` — partial match (e.g. title matches but authors differ) +- `NOT_FOUND` — no match in any database +- `ERROR` — API timeout or network failure + +**Ranking order:** overall accuracy → year compliance rate → avg confidence → completeness + +## Output files + +``` +evaluation_results/ref_hallucination_arena/ +├── evaluation_report.md # Detailed Markdown report +├── evaluation_results.json # Rankings, per-field accuracy, scores +├── verification_chart.png # Per-field accuracy bar chart +├── discipline_chart.png # Per-discipline accuracy chart +├── queries.json # Loaded evaluation queries +├── responses.json # Raw model responses +├── extracted_refs.json # Extracted BibTeX references +├── verification_results.json # Per-reference verification details +└── checkpoint.json # Pipeline checkpoint for resume +``` + +## API key by model + +| Model prefix | Environment variable | +|-------------|---------------------| +| `gpt-*`, `o1-*`, `o3-*` | `OPENAI_API_KEY` | +| `claude-*` | `ANTHROPIC_API_KEY` | +| `qwen-*`, `dashscope/*` | `DASHSCOPE_API_KEY` | +| `deepseek-*` | `DEEPSEEK_API_KEY` | +| Custom endpoint | set `api_key` + `base_url` in config | + +## Additional resources + +- Full config examples: [cookbooks/ref_hallucination_arena/examples/](../../../cookbooks/ref_hallucination_arena/examples/) +- Documentation: [docs/validating_graders/ref_hallucination_arena.md](../../../docs/validating_graders/ref_hallucination_arena.md) +- Official dataset: [HuggingFace](https://huggingface.co/datasets/OpenJudge/ref-hallucination-arena) +- Leaderboard: [openjudge.me/leaderboard](https://openjudge.me/leaderboard) diff --git a/skills/arena-eval/README.md b/skills/arena-eval/README.md new file mode 100644 index 000000000..fb8e0be97 --- /dev/null +++ b/skills/arena-eval/README.md @@ -0,0 +1,60 @@ +# Arena Eval — model/agent arena-style comparison + +Skills for comparing multiple LLMs or agents head-to-head using OpenJudge: a +general-purpose win-rate arena for any custom task, or a citation-accuracy +arena that verifies references against real bibliographic databases. + +Each skill is a self-contained workflow in `/SKILL.md`. Start at +`00-arena-router` if you're not sure which one you need. + +## The workflows + +| # | Skill | Use it when | +|---|---|---| +| 00 | `00-arena-router` | You're not sure whether you want a generic arena or a citation-hallucination benchmark. | +| 01 | `01-auto-arena` | Zero-data, custom-task comparison: generates queries, collects responses, judges pairwise, ranks by win rate. | +| 02 | `02-ref-hallucination-arena` | Benchmarks how often models fabricate citations, verified against CrossRef/PubMed/arXiv/DBLP. | + +## Relationship between the workflows + +Both produce model rankings from pairwise/verified comparison, but differ in +what counts as "correct": `01-auto-arena` relies on LLM-judge preference for +an arbitrary task (no ground truth needed); `02-ref-hallucination-arena` +checks references against real databases, so the ranking is grounded in +verifiable fact rather than judge opinion, at the cost of being scoped to +citation recommendation only. + +`02-ref-hallucination-arena` shares its content with the `academic-eval` +suite's `03-ref-hallucination-arena` — the two copies are independently +maintained (see the root [`skills/README.md`](../README.md) for why), so this +one can lean more on "how does this model compare to others" framing while +the academic-eval copy leans on "how trustworthy are this document's +citations." + +## Dependencies + +| Skill | Cookbook | +|---|---| +| `01-auto-arena` | `cookbooks/auto_arena/` | +| `02-ref-hallucination-arena` | `cookbooks/ref_hallucination_arena/` | + +```bash +pip install py-openjudge +pip install matplotlib # chart generation, needed by both skills +``` + +## Self-contained skills + +Each `/SKILL.md` is self-contained per the Anthropic Agent Skill +protocol — it can be installed and used on its own without this README or the +rest of the suite. + +## Validating the skills themselves + +This suite has an actor+judge functional test harness, same pattern as +`eval_pipeline`: an *actor* model follows a skill to answer a realistic user +request, and a separate *judge* model grades the answer against the case's +acceptance criteria. See [`tests/arena_eval_testing_guide.md`](tests/arena_eval_testing_guide.md) +for how to run it — it reuses `eval_pipeline`'s runner (parametrized, not +duplicated) against this suite's own 10 test cases in +[`tests/arena_eval_test_cases.jsonl`](tests/arena_eval_test_cases.jsonl). diff --git a/skills/arena-eval/tests/arena_eval_skill_audit.md b/skills/arena-eval/tests/arena_eval_skill_audit.md new file mode 100644 index 000000000..bd7468b1c --- /dev/null +++ b/skills/arena-eval/tests/arena_eval_skill_audit.md @@ -0,0 +1,67 @@ +# Arena Eval Skill Audit + +Initial functional-test audit of the `arena-eval` suite, run right after the +suite was created (folder-migrated from `auto-arena` + a fresh copy of +`ref-hallucination-arena` + new `00-arena-router`). + +The results below cover the original nine cases. The later router-only +regression `arena_router_005_combined_quality_and_citations` and its routing +fix are not covered by these historical runs; re-run the updated suite to +validate them against live actor/judge models. + +## Run 1 — full set, `--repeat 1` + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir skills/arena-eval/tests/results \ + --report-prefix arena_eval +``` + +Actor: `qwen3.6-plus` · Judge: `qwen3-max` + +**Result: 9/9 pass, 0 partial, 0 fail.** + +| Case | Skill | Verdict | Score | +|---|---|---:|---:| +| `arena_router_001_custom_task_comparison` | `00-arena-router` | pass | 1.00 | +| `arena_router_002_citation_hallucination_benchmark` | `00-arena-router` | pass | 1.00 | +| `arena_router_003_better_at_recommending_papers` | `00-arena-router` | pass | 1.00 | +| `arena_router_004_single_document_not_arena` | `00-arena-router` | pass | 1.00 | +| `auto_arena_001_minimal_config_two_endpoints` | `01-auto-arena` | pass | 1.00 | +| `auto_arena_002_rerun_judge_only` | `01-auto-arena` | pass | 1.00 | +| `ref_arena_arena_001_dataset_format_check` | `02-ref-hallucination-arena` | pass | 1.00 | +| `ref_arena_arena_002_ranking_tiebreak_order` | `02-ref-hallucination-arena` | pass | 1.00 | +| `arena_eval_001_end_to_end` | `arena_eval_collection` | pass | 1.00 | + +## Run 2 — variance check, `--repeat 3` on the three most ambiguity-prone cases + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir /tmp/arena_eval_repeat_check --report-prefix arena_eval \ + --case-id arena_router_003_better_at_recommending_papers \ + --case-id arena_router_004_single_document_not_arena \ + --case-id ref_arena_arena_002_ranking_tiebreak_order \ + --repeat 3 +``` + +- `arena_router_003_better_at_recommending_papers`: pass / pass / pass — score 1.00 (n=3) +- `arena_router_004_single_document_not_arena`: pass / pass / pass — score 1.00 (n=3) +- `ref_arena_arena_002_ranking_tiebreak_order`: pass / pass / pass — score 1.00 (n=3) + +No variance observed across 3 runs on any of the three boundary cases, +including the deliberately adversarial "better" phrasing case designed to +tempt a router into judge-preference framing over the verifiable workflow. + +## Reading + +Same reading as `academic-eval` (see +[`../../academic-eval/tests/academic_eval_skill_audit.md`](../../academic-eval/tests/academic_eval_skill_audit.md#reading)): +a clean 15/15 pass across both full runs and repeat checks is a good initial +signal but was authored and graded in the same sitting as the skill content +being tested, so it should be treated as a smoke signal, not a substitute for +independently authored adversarial cases or a different actor/judge pairing +down the line. diff --git a/skills/arena-eval/tests/arena_eval_test_cases.jsonl b/skills/arena-eval/tests/arena_eval_test_cases.jsonl new file mode 100644 index 000000000..48eca3027 --- /dev/null +++ b/skills/arena-eval/tests/arena_eval_test_cases.jsonl @@ -0,0 +1,10 @@ +{"id": "arena_router_001_custom_task_comparison", "skill": "00-arena-router", "category": "routing", "user_request": "I have 3 variants of my customer-support chatbot and want to know which one gives the best responses, using an LLM judge.", "artifacts": {}, "expected_behavior": ["Recognize a generic custom-task comparison.", "Recommend `01-auto-arena`."], "acceptance_criteria": ["Recommends `01-auto-arena`.", "Does not recommend `02-ref-hallucination-arena` for a generic quality comparison with no citation angle."], "failure_signals": ["Recommends `02-ref-hallucination-arena` when citations/references are not mentioned at all."]} +{"id": "arena_router_002_citation_hallucination_benchmark", "skill": "00-arena-router", "category": "routing", "user_request": "Which of these 5 models fabricates the fewest fake citations when asked to recommend academic papers?", "artifacts": {}, "expected_behavior": ["Recognize this is specifically about citation/reference hallucination.", "Recommend `02-ref-hallucination-arena`."], "acceptance_criteria": ["Recommends `02-ref-hallucination-arena`.", "Does not recommend `01-auto-arena` for this citation-specific request."], "failure_signals": ["Recommends `01-auto-arena` despite the request being specifically about citation fabrication."]} +{"id": "arena_router_003_better_at_recommending_papers", "skill": "00-arena-router", "category": "routing", "user_request": "Which model is just generally 'better' at recommending real papers to cite?", "artifacts": {}, "expected_behavior": ["Even though phrased as 'better' (typically an auto-arena style judge-preference question), recognize the domain is citation recommendation and prefer the verifiable workflow.", "Recommend `02-ref-hallucination-arena` over `01-auto-arena` because ground-truth verification is possible here."], "acceptance_criteria": ["Recommends `02-ref-hallucination-arena`.", "Explains that citation accuracy can be objectively verified rather than relying on judge opinion."], "failure_signals": ["Recommends `01-auto-arena` just because the user said 'better' instead of 'hallucinates less'."]} +{"id": "arena_router_004_single_document_not_arena", "skill": "00-arena-router", "category": "routing", "user_request": "Can you check if the references in my one paper's bibliography are legit? I'm not comparing multiple models.", "artifacts": {}, "expected_behavior": ["Recognize this is a single-document check, not a multi-model arena comparison.", "Redirect to the academic-eval suite's `02-bib-verify` (or `01-paper-review`) instead of forcing it into arena-eval."], "acceptance_criteria": ["States this isn't an arena-eval task.", "Points to academic-eval's `02-bib-verify` or `01-paper-review`."], "failure_signals": ["Tries to force this into `01-auto-arena` or `02-ref-hallucination-arena` despite there being only one document and no model comparison."]} +{"id": "arena_router_005_combined_quality_and_citations", "skill": "00-arena-router", "category": "routing", "user_request": "Compare my three models on both general helpfulness and how often they fabricate academic citations. Which workflows should I run, and in what order?", "artifacts": {}, "expected_behavior": ["Recognize two distinct evaluation goals from the request without asking the user to choose only one.", "Recommend 01-auto-arena followed by 02-ref-hallucination-arena as separate evaluations."], "acceptance_criteria": ["Names both 01-auto-arena and 02-ref-hallucination-arena, in that order.", "Explains that the first measures judge preference and the second verifies citations against bibliographic databases.", "Keeps the two results separate instead of claiming either ranking covers both goals.", "Does not ask an unnecessary diagnostic question when both goals are explicit."], "failure_signals": ["Recommends exactly one workflow and drops the other requested goal.", "Asks the user to choose between helpfulness and citation accuracy.", "Claims one run or one ranking covers both evaluation goals."]} +{"id": "auto_arena_001_minimal_config_two_endpoints", "skill": "01-auto-arena", "category": "usage", "user_request": "I want to compare gpt-4 and qwen-max on a customer-service chatbot task. I have both API keys. Help me set up the run.", "artifacts": {}, "expected_behavior": ["Produce a minimal config with task.description, two target_endpoints, and a judge_endpoint.", "Use the CLI (python -m cookbooks.auto_arena --config config.yaml --save) to run it."], "acceptance_criteria": ["Config includes a task.description field.", "Config defines both target endpoints under target_endpoints with base_url/api_key/model.", "Config defines a judge_endpoint.", "Gives the correct CLI invocation with --save."], "failure_signals": ["Omits the judge_endpoint from the config.", "Uses an invalid or nonexistent CLI flag."]} +{"id": "auto_arena_002_rerun_judge_only", "skill": "01-auto-arena", "category": "usage", "user_request": "I already ran the full arena eval with gpt-4 as judge. Now I want to see results with qwen-max as judge instead, without regenerating queries or responses.", "artifacts": {}, "expected_behavior": ["Recognize this maps to the --rerun-judge flag, which keeps queries/responses/rubrics and only re-runs pairwise evaluation.", "Not recommend --fresh, which would discard everything."], "acceptance_criteria": ["Recommends the --rerun-judge flag.", "Explicitly states queries, responses, and rubrics are preserved.", "Does not recommend --fresh."], "failure_signals": ["Recommends --fresh, which would needlessly regenerate queries and responses.", "Fails to mention the --rerun-judge flag at all."]} +{"id": "ref_arena_arena_001_dataset_format_check", "skill": "02-ref-hallucination-arena", "category": "dataset_design", "user_request": "I have 200 questions asking for paper recommendations, but they're just plain text strings in a .txt file, one per line. Can I use this directly with the arena benchmark?", "artifacts": {}, "expected_behavior": ["Explain the required dataset format: JSON/JSONL with a query field, plus optional discipline, num_refs, language, year_constraint.", "State that a plain .txt file must first be converted."], "acceptance_criteria": ["Mentions the dataset must be JSON or JSONL, not plain .txt.", "Lists query as a required field.", "Does not claim the .txt file works as-is."], "failure_signals": ["Claims the .txt works directly without conversion."]} +{"id": "ref_arena_arena_002_ranking_tiebreak_order", "skill": "02-ref-hallucination-arena", "category": "interpretation", "user_request": "Two models tied on overall accuracy in my benchmark. How do I decide which one ranks higher?", "artifacts": {}, "expected_behavior": ["Apply the documented ranking order: overall accuracy -> year compliance rate -> avg confidence -> completeness.", "Use year compliance rate as the first tiebreaker."], "acceptance_criteria": ["States year compliance rate is the first tiebreaker after overall accuracy.", "Mentions avg confidence and/or completeness as further tiebreakers if still tied."], "failure_signals": ["Picks a ranking method not documented in the skill (e.g. random, alphabetical).", "Fails to mention any tiebreaker at all."]} +{"id": "arena_eval_001_end_to_end", "skill": "arena_eval_collection", "category": "end_to_end", "user_request": "I want a full picture: compare 3 models generally on helpfulness AND specifically on how often they hallucinate citations. What do I run, in what order, and why two different workflows?", "artifacts": {}, "expected_behavior": ["Recognize two distinct comparisons are needed.", "Recommend running both 01-auto-arena (general helpfulness) and 02-ref-hallucination-arena (citation hallucination) as separate runs, explaining why one judge-based ranking can't cover both."], "acceptance_criteria": ["Names both 01-auto-arena and 02-ref-hallucination-arena.", "Explains they measure different things (judge preference vs verified citation accuracy) and can't be collapsed into one run."], "failure_signals": ["Suggests only one of the two skills covers everything.", "Conflates general helpfulness with citation accuracy."]} diff --git a/skills/arena-eval/tests/arena_eval_test_cases.md b/skills/arena-eval/tests/arena_eval_test_cases.md new file mode 100644 index 000000000..cfbc3cf70 --- /dev/null +++ b/skills/arena-eval/tests/arena_eval_test_cases.md @@ -0,0 +1,21 @@ +# Arena Eval Functional Test Cases (Summary) + +Source fixture: [`arena_eval_test_cases.jsonl`](arena_eval_test_cases.jsonl) +(the runner reads only the JSONL; this table is for human review). 10 cases +across the suite's 3 skills. + +| ID | Skill | Category | Request (abridged) | Key acceptance criterion | +|---|---|---|---|---| +| `arena_router_001_custom_task_comparison` | `00-arena-router` | routing | Compare 3 chatbot variants, generic quality | Routes to `01-auto-arena` | +| `arena_router_002_citation_hallucination_benchmark` | `00-arena-router` | routing | Which model fabricates fewest citations | Routes to `02-ref-hallucination-arena` | +| `arena_router_003_better_at_recommending_papers` | `00-arena-router` | routing | "Better" at recommending papers (ambiguous phrasing) | Still routes to `02-ref-hallucination-arena` (verifiable > judge opinion) | +| `arena_router_004_single_document_not_arena` | `00-arena-router` | routing | Check one paper's own bibliography | Redirects to `academic-eval` | +| `arena_router_005_combined_quality_and_citations` | `00-arena-router` | routing | Compare helpfulness AND citation fabrication | Router alone recommends both workflows in order, without dropping either goal | +| `auto_arena_001_minimal_config_two_endpoints` | `01-auto-arena` | usage | Compare gpt-4 vs qwen-max, have both keys | Config has `task`, both `target_endpoints`, `judge_endpoint` | +| `auto_arena_002_rerun_judge_only` | `01-auto-arena` | usage | Swap judge model, keep queries/responses | Recommends `--rerun-judge`, not `--fresh` | +| `ref_arena_arena_001_dataset_format_check` | `02-ref-hallucination-arena` | dataset_design | Plain `.txt` queries | Requires JSON/JSONL with `query` field | +| `ref_arena_arena_002_ranking_tiebreak_order` | `02-ref-hallucination-arena` | interpretation | Two models tied on accuracy | Applies documented tiebreak order (year compliance first) | +| `arena_eval_001_end_to_end` | `arena_eval_collection` | end_to_end | Compare on helpfulness AND citation hallucination | Names both `01-auto-arena` and `02-ref-hallucination-arena` | + +See [`arena_eval_testing_guide.md`](arena_eval_testing_guide.md) for how to +run these and interpret results. diff --git a/skills/arena-eval/tests/arena_eval_testing_guide.md b/skills/arena-eval/tests/arena_eval_testing_guide.md new file mode 100644 index 000000000..aeae813a6 --- /dev/null +++ b/skills/arena-eval/tests/arena_eval_testing_guide.md @@ -0,0 +1,129 @@ +# Arena Eval Functional Testing Guide + +This guide explains how to prove the `arena-eval` skills work on realistic +scenarios. It follows the same actor+judge functional-test pattern as +[`skills/eval_pipeline/tests/`](../../eval_pipeline/tests/eval_pipeline_testing_guide.md) — +this suite reuses that suite's runner rather than shipping a duplicate one +(see [Why a shared runner](#why-a-shared-runner) below, same rationale as +`academic-eval`'s guide). + +## What To Test + +1. Skill package quality test — `cookbooks/skills_evaluation/evaluate_skills.py`. +2. Functional scenario test (this guide) — `skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py` + pointed at `skills/arena-eval`. Checks whether a skill guides an agent to the + right recommendation (routing) or the right concrete answer (config/flags/ + interpretation) for a realistic user request. + +## API Key + +```bash +# .env at repo root — Aliyun DashScope (Bailian) +DASHSCOPE_API_KEY=sk-... + +# or, any OpenAI-compatible provider +export OPENAI_API_KEY="sk-..." +export OPENAI_BASE_URL="https://your-provider.example/v1" # omit for OpenAI itself +``` + +Actor and judge default to different models (`qwen3.6-plus` / `qwen3-max`) so +one model doesn't grade its own output. + +## Quick Smoke Test + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir skills/arena-eval/tests/results \ + --report-prefix arena_eval \ + --case-id arena_router_002_citation_hallucination_benchmark +``` + +Representative smoke set: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir skills/arena-eval/tests/results \ + --report-prefix arena_eval \ + --case-id arena_router_003_better_at_recommending_papers \ + --case-id arena_router_004_single_document_not_arena \ + --case-id arena_router_005_combined_quality_and_citations \ + --case-id auto_arena_002_rerun_judge_only \ + --case-id ref_arena_arena_002_ranking_tiebreak_order \ + --case-id arena_eval_001_end_to_end +``` + +Full set: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir skills/arena-eval/tests/results \ + --report-prefix arena_eval +``` + +Outputs: + +```text +skills/arena-eval/tests/results/arena_eval_functional_report.md +skills/arena-eval/tests/results/arena_eval_functional_results.json +``` + +## Recommended Test Samples + +| Purpose | Case ID | Why | +|---|---|---| +| Router: citation-specific | `arena_router_002_citation_hallucination_benchmark` | Ensures citation-fabrication requests go to `02-ref-hallucination-arena`, not `01-auto-arena`. | +| Router: "better" trap | `arena_router_003_better_at_recommending_papers` | Ensures phrasing as "better" (not "hallucinates less") still routes to the verifiable workflow. | +| Router: out-of-scope redirect | `arena_router_004_single_document_not_arena` | Ensures a single-document (not multi-model) request is redirected to `academic-eval`. | +| Router: combined request | `arena_router_005_combined_quality_and_citations` | Loads only the router to check that both goals are preserved without help from the sub-skills. | +| Auto arena: flag choice | `auto_arena_002_rerun_judge_only` | Ensures `--rerun-judge` is recommended over `--fresh` when only the judge changes. | +| Ref arena: tiebreak | `ref_arena_arena_002_ranking_tiebreak_order` | Ensures the documented tiebreak order is used, not an arbitrary one. | +| End-to-end | `arena_eval_001_end_to_end` | Ensures a request spanning both workflows isn't collapsed into one run. | + +## Handling Variance + +Use `--repeat 3` before claiming a skill change worked — actor and judge are +both LLMs, so single-run verdicts near a boundary are noisy. + +For the combined-request regression, run the router-only case with repeats: + +```bash +python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \ + --skill-root skills/arena-eval \ + --cases skills/arena-eval/tests/arena_eval_test_cases.jsonl \ + --out-dir skills/arena-eval/tests/results \ + --report-prefix arena_eval \ + --case-id arena_router_005_combined_quality_and_citations --repeat 3 +``` + +## How To Interpret Results + +- `pass`: all critical acceptance criteria met. +- `partial`: mostly correct but missed a flag, field, or caveat. +- `fail`: routed incorrectly or gave a misleading answer. + +Useful pass threshold for a first audit: smoke set 6/6 pass or partial with at +least 5 pass; full set at least 80% pass. + +## What To Do After A Failure + +1. Read the actor output in the Markdown report. +2. Compare it to the missed acceptance criteria. +3. Patch the relevant `SKILL.md` (or the router's triage table / key-distinction + section) with a clearer rule. +4. Re-run only that case, then the smoke set. + +## Why a shared runner + +Same rationale as `academic-eval`: the runner at +`skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py` is parametrized +via `--skill-root` / `--cases` / `--out-dir` / `--report-prefix` and isn't +subject to the Agent Skill protocol's independent-installability constraint +(that constraint is about `SKILL.md` content, not internal test tooling), so +copying it per suite would only create three scripts to keep in sync instead +of one. diff --git a/skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py b/skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py index 035b7b62c..bf763a5fb 100644 --- a/skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py +++ b/skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py @@ -1,16 +1,30 @@ # -*- coding: utf-8 -*- """ -Functional tests for eval-pipeline skills. +Functional tests for Agent Skill suites (actor + judge, dual-model). This runner checks whether a skill can guide an agent through realistic user requests. It is different from ``evaluate_skills.py``, which grades the quality of a skill package itself. +Despite the filename (kept for backward compatibility — it originated in +``eval_pipeline`` and is still that suite's default target), this runner is +suite-agnostic: point it at any ``skills//`` directory with +``--skill-root`` and its own JSONL fixture with ``--cases``. Every default +below still resolves to ``eval_pipeline`` when no flags are given, so existing +``eval_pipeline`` invocations are unaffected. + Usage: python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \\ --case-id rag_eval_001_diagnose_generation_problem + # Running against a different suite (e.g. academic-eval): + python skills/eval_pipeline/tests/run_eval_pipeline_skill_tests.py \\ + --skill-root skills/academic-eval \\ + --cases skills/academic-eval/tests/academic_eval_test_cases.jsonl \\ + --out-dir skills/academic-eval/tests/results \\ + --report-prefix academic_eval + Environment (any OpenAI-compatible provider): OPENAI_API_KEY preferred; the OpenAI-compatible key OPENAI_BASE_URL optional, for OpenAI-compatible providers @@ -80,7 +94,11 @@ def with_bundled_refs(skill_dir: Path, text: str) -> str: extra.append(ref.read_text(encoding="utf-8")) return text + "".join(extra) - if skill_id == "eval_pipeline_collection": + # A skill id ending in "_collection" (e.g. "eval_pipeline_collection", + # "academic_eval_collection") is a sentinel meaning "load every SKILL.md + # under this suite's skill_root", used for end-to-end test cases that span + # a router + its sub-skills rather than a single one. + if skill_id.endswith("_collection"): parts = [] for skill_file in sorted(skill_root.glob("*/SKILL.md")): parts.append(f"\n\n===== {skill_file.parent.name} =====\n") @@ -192,7 +210,7 @@ def run_actor(client: OpenAI, model: str, skill_text: str, case: dict[str, Any]) def judge_actor_output(client: OpenAI, model: str, case: dict[str, Any], actor_output: str) -> dict[str, Any]: system = ( - "You are a strict evaluator for eval-pipeline Skill functional tests. " + "You are a strict evaluator for Agent Skill functional tests. " "Grade only against the acceptance criteria and failure signals. " "Return only a JSON object." ) @@ -238,9 +256,9 @@ def judge_actor_output(client: OpenAI, model: str, case: dict[str, Any], actor_o return result -def render_report(results: list[dict[str, Any]]) -> str: +def render_report(results: list[dict[str, Any]], title: str) -> str: lines = [ - "# Eval Pipeline Functional Test Report", + f"# {title} Functional Test Report", "", f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}", "", @@ -315,10 +333,17 @@ def aggregate_runs(runs: list[dict[str, Any]]) -> tuple[str, dict[str, Any]]: def main() -> None: - parser = argparse.ArgumentParser(description="Run functional tests for eval-pipeline skills.") + parser = argparse.ArgumentParser(description="Run functional tests for an Agent Skill suite (actor + judge).") parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) parser.add_argument("--skill-root", type=Path, default=DEFAULT_SKILL_ROOT) parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR) + parser.add_argument( + "--report-prefix", + default=None, + help="Prefix for output filenames and report title, e.g. 'academic_eval' -> " + "academic_eval_functional_report.md. Defaults to --skill-root's directory name " + "(underscored), which is 'eval_pipeline' when --skill-root is left at its default.", + ) parser.add_argument("--case-id", action="append", help="Run only the given case id. Can be repeated.") parser.add_argument("--limit", type=int, default=None, help="Limit number of cases after filtering.") parser.add_argument( @@ -337,6 +362,7 @@ def main() -> None: "Single-run verdicts are noisy (LLM actor + judge); use >=3 for robust claims.", ) args = parser.parse_args() + report_prefix = args.report_prefix or args.skill_root.name.replace("-", "_") load_dotenv(REPO_ROOT / ".env") client = build_client() @@ -373,10 +399,11 @@ def main() -> None: results.append({"case": case, "actor_output": actor_output, "judge": judge}) args.out_dir.mkdir(parents=True, exist_ok=True) - json_path = args.out_dir / "eval_pipeline_functional_results.json" - md_path = args.out_dir / "eval_pipeline_functional_report.md" + json_path = args.out_dir / f"{report_prefix}_functional_results.json" + md_path = args.out_dir / f"{report_prefix}_functional_report.md" + report_title = report_prefix.replace("_", " ").title() json_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") - md_path.write_text(render_report(results), encoding="utf-8") + md_path.write_text(render_report(results, report_title), encoding="utf-8") passed = sum(1 for item in results if item["judge"].get("verdict") == "pass") partial = sum(1 for item in results if item["judge"].get("verdict") == "partial") diff --git a/skills/openjudge/SKILL.md b/skills/openjudge-core/01-graders-and-pipeline/SKILL.md similarity index 99% rename from skills/openjudge/SKILL.md rename to skills/openjudge-core/01-graders-and-pipeline/SKILL.md index 44e0eb8db..0efe9b782 100644 --- a/skills/openjudge/SKILL.md +++ b/skills/openjudge-core/01-graders-and-pipeline/SKILL.md @@ -1,5 +1,5 @@ --- -name: openjudge +name: 01-graders-and-pipeline description: > Build custom LLM evaluation pipelines using the OpenJudge framework. Covers selecting and configuring graders (LLM-based, function-based, agentic), diff --git a/skills/openjudge/analyzer.md b/skills/openjudge-core/01-graders-and-pipeline/analyzer.md similarity index 100% rename from skills/openjudge/analyzer.md rename to skills/openjudge-core/01-graders-and-pipeline/analyzer.md diff --git a/skills/openjudge/generator.md b/skills/openjudge-core/01-graders-and-pipeline/generator.md similarity index 100% rename from skills/openjudge/generator.md rename to skills/openjudge-core/01-graders-and-pipeline/generator.md diff --git a/skills/openjudge/graders.md b/skills/openjudge-core/01-graders-and-pipeline/graders.md similarity index 100% rename from skills/openjudge/graders.md rename to skills/openjudge-core/01-graders-and-pipeline/graders.md diff --git a/skills/openjudge/pipeline.md b/skills/openjudge-core/01-graders-and-pipeline/pipeline.md similarity index 100% rename from skills/openjudge/pipeline.md rename to skills/openjudge-core/01-graders-and-pipeline/pipeline.md diff --git a/skills/rl-reward/SKILL.md b/skills/openjudge-core/02-rl-reward/SKILL.md similarity index 99% rename from skills/rl-reward/SKILL.md rename to skills/openjudge-core/02-rl-reward/SKILL.md index d3da6f5f6..0eb5c4893 100644 --- a/skills/rl-reward/SKILL.md +++ b/skills/openjudge-core/02-rl-reward/SKILL.md @@ -1,5 +1,5 @@ --- -name: rl-reward +name: 02-rl-reward description: > Build RL reward signals using the OpenJudge framework. Covers choosing between pointwise and pairwise reward strategies based on diff --git a/skills/rl-reward/pairwise.md b/skills/openjudge-core/02-rl-reward/pairwise.md similarity index 100% rename from skills/rl-reward/pairwise.md rename to skills/openjudge-core/02-rl-reward/pairwise.md diff --git a/skills/rl-reward/pointwise.md b/skills/openjudge-core/02-rl-reward/pointwise.md similarity index 100% rename from skills/rl-reward/pointwise.md rename to skills/openjudge-core/02-rl-reward/pointwise.md diff --git a/skills/openjudge-core/README.md b/skills/openjudge-core/README.md new file mode 100644 index 000000000..90ef15990 --- /dev/null +++ b/skills/openjudge-core/README.md @@ -0,0 +1,54 @@ +# OpenJudge Core — build evaluation pipelines & RL reward signals + +Skills that teach the core OpenJudge library API directly: selecting and +running graders over a dataset, and turning grader output into reward signals +for RL training. + +Each skill is a self-contained workflow in `/SKILL.md`. + +## The workflows + +| # | Skill | Use it when | +|---|---|---| +| 01 | `01-graders-and-pipeline` | You want to evaluate LLM outputs: pick/configure graders, run batch evaluation with `GradingRunner`, aggregate scores, auto-generate graders, analyze results. | +| 02 | `02-rl-reward` | You want to build RL reward signals: pointwise multi-dimensional rewards, pairwise tournament rewards for GRPO, preference pairs for DPO/RLAIF. | + +## No router — and why + +Unlike `academic-eval`/`arena-eval`, this suite has **no `00-xxx-router`**. +The two skills' scopes don't overlap: "build an evaluation pipeline" and +"build an RL reward signal" are different keywords, different user intents, +and different entry points into the OpenJudge API. Each skill's `description` +frontmatter is specific enough for client-side semantic routing (Cursor, +Claude Code, Codex, etc.) to pick the right one without an extra triage step. +If you're building an RL reward on top of an evaluation pipeline you already +built with `01`, just read `02` next — no diagnostic question needed. + +## Sub-documents + +Both skills ship topic-specific sub-documents in their own directory (read +inline from the `SKILL.md`, not through this README): + +| Skill | Sub-doc | Topic | +|---|---|---| +| `01-graders-and-pipeline` | `graders.md` | Grader selection & configuration | +| `01-graders-and-pipeline` | `pipeline.md` | Batch evaluation pipeline | +| `01-graders-and-pipeline` | `generator.md` | Auto-generate graders from labeled data | +| `01-graders-and-pipeline` | `analyzer.md` | Analyze & compare results (win rates, stats) | +| `02-rl-reward` | `pointwise.md` | Pointwise multi-dimensional reward | +| `02-rl-reward` | `pairwise.md` | Pairwise reward (tournament / DPO preference pairs) | + +## Dependency + +```bash +pip install py-openjudge +``` + +Both skills teach the `openjudge` core library directly (`openjudge.graders`, +`openjudge.runner`, `openjudge.evaluation_strategy`) — no cookbook dependency. + +## Self-contained skills + +Each `/SKILL.md` (with its sub-documents) is self-contained per the +Anthropic Agent Skill protocol — it can be installed and used on its own +without this README or the rest of the suite. diff --git a/tests/cookbooks/test_skill_loader.py b/tests/cookbooks/test_skill_loader.py new file mode 100644 index 000000000..5f77217fc --- /dev/null +++ b/tests/cookbooks/test_skill_loader.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +"""Offline regressions for skill discovery and domain-suite packaging.""" + +from pathlib import Path + +import pytest + +from cookbooks.skills_evaluation.skill_models import SkillLoader + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOMAIN_SUITES = ("academic-eval", "arena-eval", "openjudge-core") +DOMAIN_SKILLS = sorted(skill for suite in DOMAIN_SUITES for skill in (REPO_ROOT / "skills" / suite).glob("*/SKILL.md")) +pytestmark = pytest.mark.unit + + +def _write_skill(directory: Path) -> Path: + directory.mkdir(parents=True) + (directory / "SKILL.md").write_text( + f"---\nname: {directory.name}\ndescription: A test skill.\n---\n\nTest instructions.\n", + encoding="utf-8", + ) + return directory + + +def test_loads_flat_and_nested_skills(tmp_path: Path) -> None: + nested = _write_skill(tmp_path / "a-suite" / "subgroup" / "nested") + standalone = _write_skill(tmp_path / "standalone") + (tmp_path / "empty-suite").mkdir() + + skills = SkillLoader.load_from_directory(tmp_path) + + assert [skill.directory for skill in skills] == [nested, standalone] + + +@pytest.mark.parametrize("single_skill", [True, False]) +def test_stops_discovery_at_package_boundary(tmp_path: Path, single_skill: bool) -> None: + package = _write_skill(tmp_path / "package") + _write_skill(package / "references" / "example") + + skills = SkillLoader.load_from_directory(package if single_skill else tmp_path) + + assert [skill.directory for skill in skills] == [package] + assert "references/example/SKILL.md" in {file.relative_path for file in skills[0].files} + + +@pytest.mark.parametrize("ignored", [".git", ".venv", "node_modules", "__pycache__"]) +def test_ignores_tooling_directories_inside_suites(tmp_path: Path, ignored: str) -> None: + real = _write_skill(tmp_path / "suite" / "real") + _write_skill(tmp_path / "suite" / ignored / "example") + + assert [skill.directory for skill in SkillLoader.load_from_directory(tmp_path)] == [real] + + +def test_does_not_discover_examples_in_an_invalid_package(tmp_path: Path) -> None: + invalid = _write_skill(tmp_path / "suite" / "invalid") + (invalid / "SKILL.md").write_text("Not a valid skill manifest.\n", encoding="utf-8") + _write_skill(invalid / "references" / "example") + + assert SkillLoader.load_from_directory(tmp_path) == [] + + +def test_symlink_cycles_and_aliases_do_not_duplicate_skills(tmp_path: Path) -> None: + real = _write_skill(tmp_path / "suite" / "real") + (tmp_path / "suite" / "cycle").symlink_to(tmp_path, target_is_directory=True) + (tmp_path / "alias").symlink_to(real, target_is_directory=True) + + skills = SkillLoader.load_from_directory(tmp_path) + + assert len(skills) == 1 + assert skills[0].directory.resolve() == real.resolve() + + +def test_rejects_non_directory_input(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Not a directory"): + SkillLoader.load_from_directory(tmp_path / "missing") + + +def test_repository_root_includes_every_domain_suite() -> None: + skills = SkillLoader.load_from_directory(REPO_ROOT / "skills") + discovered = {skill.skill_md_path for skill in skills} + + assert DOMAIN_SKILLS + assert set(DOMAIN_SKILLS) <= discovered + assert REPO_ROOT / "skills" / "mmx-cli" / "SKILL.md" in discovered + assert REPO_ROOT / "skills" / "eval_pipeline" / "00-meta-eval" / "SKILL.md" in discovered + + +@pytest.mark.parametrize("skill_md", DOMAIN_SKILLS, ids=lambda path: str(path.relative_to(REPO_ROOT / "skills"))) +def test_domain_skill_name_matches_install_directory(skill_md: Path) -> None: + skill = SkillLoader.load_skill(skill_md.parent) + + assert skill is not None + assert skill.manifest.name == skill_md.parent.name + + +def test_domain_skill_names_do_not_collide() -> None: + skills = [SkillLoader.load_skill(path.parent) for path in DOMAIN_SKILLS] + names = [skill.manifest.name for skill in skills] + + assert len(set(names)) == len(names)