diff --git a/.claude/rules/cli-renames.md b/.claude/rules/cli-renames.md index a6dec42122..2e3cb8db70 100644 --- a/.claude/rules/cli-renames.md +++ b/.claude/rules/cli-renames.md @@ -15,6 +15,7 @@ The catalog at `assets/uip-catalog-snapshot.json` is the source of truth for whi | `functions` | `function` | uip 1.200.0| | `sc` | `aops` | uip 1.201.0| | `cicd` | `aops` | uip 1.201.0| +| `or eval` | `eval` | uip 1.202.0| ## When to add an entry diff --git a/assets/uip-catalog-snapshot.json b/assets/uip-catalog-snapshot.json index 54a89dee5f..3a578545fa 100644 --- a/assets/uip-catalog-snapshot.json +++ b/assets/uip-catalog-snapshot.json @@ -412,6 +412,38 @@ "df records update", "docsai", "docsai ask", + "eval", + "eval eval-set", + "eval eval-set create", + "eval eval-set delete", + "eval eval-set get", + "eval eval-set list", + "eval eval-set update", + "eval evaluation", + "eval evaluation create", + "eval evaluation delete", + "eval evaluation get", + "eval evaluation list", + "eval evaluation update", + "eval evaluator", + "eval evaluator create", + "eval evaluator delete", + "eval evaluator get", + "eval evaluator list", + "eval evaluator update", + "eval execute-and-evaluate", + "eval run", + "eval run get", + "eval run list", + "eval run results", + "eval schedule", + "eval schedule create", + "eval schedule delete", + "eval schedule get", + "eval schedule list", + "eval schedule pause", + "eval schedule resume", + "eval schedule update", "feedback", "feedback send", "function", @@ -996,38 +1028,6 @@ "or credential-stores", "or credential-stores get", "or credential-stores list", - "or eval", - "or eval eval-set", - "or eval eval-set create", - "or eval eval-set delete", - "or eval eval-set get", - "or eval eval-set list", - "or eval eval-set update", - "or eval evaluation", - "or eval evaluation create", - "or eval evaluation delete", - "or eval evaluation get", - "or eval evaluation list", - "or eval evaluation update", - "or eval evaluator", - "or eval evaluator create", - "or eval evaluator delete", - "or eval evaluator get", - "or eval evaluator list", - "or eval evaluator update", - "or eval execute-and-evaluate", - "or eval run", - "or eval run get", - "or eval run list", - "or eval run results", - "or eval schedule", - "or eval schedule create", - "or eval schedule delete", - "or eval schedule get", - "or eval schedule list", - "or eval schedule pause", - "or eval schedule resume", - "or eval schedule update", "or feeds", "or feeds list", "or folders", diff --git a/skills/uipath-agents/SKILL.md b/skills/uipath-agents/SKILL.md index acb99bb710..da5597d557 100644 --- a/skills/uipath-agents/SKILL.md +++ b/skills/uipath-agents/SKILL.md @@ -69,7 +69,7 @@ Determine the agent mode before proceeding: | Check, validate, diagnose, or fix whether an existing coded guardrail is correctly configured (placement / scope) | Coded | [coded/capabilities/guardrails/guardrails-recommend.md](references/coded/capabilities/guardrails/guardrails-recommend.md) § Validate Mode | **fetch SDK docs first (authoritative for scope/placement)**; also fetch catalog + list for relevance/entitlement — then check correctness, actionability, and relevance — fix in place | | Embed a low-code agent inline in a flow, or wire a multi-agent solution | Low-code | [lowcode/lowcode.md](references/lowcode/lowcode.md) § Capability Registry | `lowcode/capabilities/inline-in-flow/inline-in-flow.md`, `lowcode/capabilities/process/solution-agent.md` | | Run low-code evaluations | Low-code | [lowcode/evaluations/evaluate.md](references/lowcode/evaluations/evaluate.md) | `lowcode/evaluations/evaluators.md`, `lowcode/evaluations/evaluation-sets.md`, `lowcode/evaluations/running-evaluations.md` | -| Manage runtime evals for a published Orchestrator package | Low-code | [lowcode/evaluations/orchestrator-eval-run.md](references/lowcode/evaluations/orchestrator-eval-run.md) | `uip or eval execute-and-evaluate`, evaluator/eval-set/evaluation CRUD, run results, schedules | +| Manage runtime evals for a published Orchestrator package | Low-code | [lowcode/evaluations/orchestrator-eval-run.md](references/lowcode/evaluations/orchestrator-eval-run.md) | `uip eval execute-and-evaluate`, evaluator/eval-set/evaluation CRUD, run results, schedules | | Validate, pack, publish, upload, or deploy a low-code agent | Low-code | [lowcode/lowcode.md](references/lowcode/lowcode.md) | `lowcode/project-lifecycle.md`, `lowcode/solution-resources.md` | | Debug / run a low-code agent end-to-end and inspect its output | Low-code | [lowcode/debug.md](references/lowcode/debug.md) | `lowcode/project-lifecycle.md` § `uip agent debug` | | Embed coded agent in a flow (solution-level) | Coded | [coded/embedding-in-flows.md](references/coded/embedding-in-flows.md) | | diff --git a/skills/uipath-agents/references/lowcode/evaluations/evaluate.md b/skills/uipath-agents/references/lowcode/evaluations/evaluate.md index c53f73501b..2f55cc54d5 100644 --- a/skills/uipath-agents/references/lowcode/evaluations/evaluate.md +++ b/skills/uipath-agents/references/lowcode/evaluations/evaluate.md @@ -31,7 +31,7 @@ Local operations (managing evaluators, eval sets, test cases) do **not** require - [Evaluators](evaluators.md) — evaluator types, adding/removing, default prompts - [Evaluation Sets and Test Cases](evaluation-sets.md) — creating sets, adding test cases, simulation options - [Running Evaluations](running-evaluations.md) — start, status, results, compare -- [Orchestrator Runtime Eval Commands](orchestrator-eval-run.md) — full CRUD for evaluators, eval sets, data points, run/schedule/results against published Orchestrator packages +- [Runtime Eval Commands](orchestrator-eval-run.md) — full CRUD for evaluators, eval sets, data points, run/schedule/results via `uip eval` Read Evaluators before choosing an evaluator type, and Evaluation Sets before writing test cases. diff --git a/skills/uipath-agents/references/lowcode/evaluations/orchestrator-eval-run.md b/skills/uipath-agents/references/lowcode/evaluations/orchestrator-eval-run.md index 1b996b32d1..dd0e924e99 100644 --- a/skills/uipath-agents/references/lowcode/evaluations/orchestrator-eval-run.md +++ b/skills/uipath-agents/references/lowcode/evaluations/orchestrator-eval-run.md @@ -5,7 +5,7 @@ Manage and run runtime evaluations for agents published as Orchestrator packages ## Command Structure ``` -uip or eval +uip eval ├── execute-and-evaluate Submit a runtime eval run ├── run list List eval set runs for a process ├── run get Get details of a specific run @@ -24,7 +24,7 @@ uip or eval Submit a runtime eval run for a published Orchestrator package. ```bash -uip or eval execute-and-evaluate \ +uip eval execute-and-evaluate \ --process-key \ --workload-id \ --items \ @@ -52,7 +52,7 @@ The folder resolves from your personal workspace automatically. Pass `--folder-k ### Example ```bash -uip or eval execute-and-evaluate \ +uip eval execute-and-evaluate \ --process-key "9e4b2f17-7c3a-4d81-b592-3f6e8a1d5c09" \ --workload-id "a1b2c3d4-0000-0000-0000-000000000001" \ --items '[{"id":"i1","name":"Test","inputs":{"input":"hello"},"expectedOutput":{},"expectedBehavior":""}]' \ @@ -84,7 +84,7 @@ CRUD for evaluators scoped by process key. ### evaluator list ```bash -uip or eval evaluator list --process-key [--limit ] [--offset ] [--tenant ] --output json +uip eval evaluator list --process-key [--limit ] [--offset ] [--tenant ] --output json ``` Output code: `EvaluatorList`. Fields: EvaluatorId, Name, Description, EvaluatorTypeId, Version, CreatedAt. Includes `Pagination` field. @@ -92,7 +92,7 @@ Output code: `EvaluatorList`. Fields: EvaluatorId, Name, Description, EvaluatorT ### evaluator get ```bash -uip or eval evaluator get --process-key [--tenant ] --output json +uip eval evaluator get --process-key [--tenant ] --output json ``` Output code: `EvaluatorDetails`. @@ -100,7 +100,7 @@ Output code: `EvaluatorDetails`. ### evaluator create ```bash -uip or eval evaluator create \ +uip eval evaluator create \ --process-key \ --workload-id \ --folder-key \ @@ -129,7 +129,7 @@ Output code: `EvaluatorCreated`. ### evaluator update ```bash -uip or eval evaluator update \ +uip eval evaluator update \ --process-key \ [--name ] \ [--description ] \ @@ -147,7 +147,7 @@ Output code: `EvaluatorUpdated`. ### evaluator delete ```bash -uip or eval evaluator delete --process-key [--tenant ] --output json +uip eval evaluator delete --process-key [--tenant ] --output json ``` Output code: `EvaluatorDeleted`. @@ -161,7 +161,7 @@ CRUD for eval sets (dataset containers) scoped by process key. ### eval-set list ```bash -uip or eval eval-set list --process-key [--limit ] [--offset ] [--tenant ] --output json +uip eval eval-set list --process-key [--limit ] [--offset ] [--tenant ] --output json ``` Output code: `EvalSetList`. Fields: EvalSetId, Name, Description, BatchSize, EvaluatorRefs, CreatedAt. Includes `Pagination` field. @@ -169,7 +169,7 @@ Output code: `EvalSetList`. Fields: EvalSetId, Name, Description, BatchSize, Eva ### eval-set get ```bash -uip or eval eval-set get --process-key [--tenant ] --output json +uip eval eval-set get --process-key [--tenant ] --output json ``` Output code: `EvalSetDetails`. @@ -177,7 +177,7 @@ Output code: `EvalSetDetails`. ### eval-set create ```bash -uip or eval eval-set create \ +uip eval eval-set create \ --process-key \ --workload-id \ --folder-key \ @@ -206,7 +206,7 @@ Output code: `EvalSetCreated`. ### eval-set update ```bash -uip or eval eval-set update \ +uip eval eval-set update \ --process-key \ [--name ] \ [--description ] \ @@ -224,7 +224,7 @@ Output code: `EvalSetUpdated`. ### eval-set delete ```bash -uip or eval eval-set delete --process-key [--tenant ] --output json +uip eval eval-set delete --process-key [--tenant ] --output json ``` Output code: `EvalSetDeleted`. @@ -238,7 +238,7 @@ CRUD for evaluations (test cases / data points) within eval sets. ### evaluation list ```bash -uip or eval evaluation list \ +uip eval evaluation list \ --process-key \ --eval-set-id \ [--limit ] \ @@ -252,7 +252,7 @@ Output code: `EvaluationList`. Fields: EvaluationId, EvalSetId, Name, Inputs, Ex ### evaluation get ```bash -uip or eval evaluation get \ +uip eval evaluation get \ --process-key \ --eval-set-id \ [--tenant ] \ @@ -264,7 +264,7 @@ Output code: `EvaluationDetails`. ### evaluation create ```bash -uip or eval evaluation create \ +uip eval evaluation create \ --process-key \ --eval-set-id \ --folder-key \ @@ -293,7 +293,7 @@ Output code: `EvaluationCreated`. ### evaluation update ```bash -uip or eval evaluation update \ +uip eval evaluation update \ --process-key \ --eval-set-id \ [--name ] \ @@ -312,7 +312,7 @@ Output code: `EvaluationUpdated`. ### evaluation delete ```bash -uip or eval evaluation delete \ +uip eval evaluation delete \ --process-key \ --eval-set-id \ [--tenant ] \ @@ -330,7 +330,7 @@ Query eval run results by process key. ### run list ```bash -uip or eval run list --process-key [--limit ] [--offset ] [--tenant ] --output json +uip eval run list --process-key [--limit ] [--offset ] [--tenant ] --output json ``` Output code: `EvalSetRunList`. Fields: EvalSetRunId, EvalSetId, Status, Score, EvalsExecuted, Duration, CreatedAt. Includes `Pagination` field. @@ -338,7 +338,7 @@ Output code: `EvalSetRunList`. Fields: EvalSetRunId, EvalSetId, Status, Score, E ### run get ```bash -uip or eval run get --process-key [--tenant ] --output json +uip eval run get --process-key [--tenant ] --output json ``` Output code: `EvalSetRunDetails`. @@ -346,7 +346,7 @@ Output code: `EvalSetRunDetails`. ### run results ```bash -uip or eval run results --process-key [--tenant ] --output json +uip eval run results --process-key [--tenant ] --output json ``` Output code: `EvalRunResults`. Fields: EvalRunId, DataPoint, Status, Result, CreatedAt. @@ -360,7 +360,7 @@ CRUD for scheduled recurring eval runs. ### schedule create ```bash -uip or eval schedule create \ +uip eval schedule create \ --process-key \ --eval-set-id \ --cron \ @@ -377,12 +377,12 @@ Output code: `EvalScheduleCreated`. Fields: ScheduleId, WorkloadId, ProcessKey, ### schedule list / get / update / pause / resume / delete ```bash -uip or eval schedule list --process-key --output json -uip or eval schedule get --process-key --output json -uip or eval schedule update --process-key [--eval-set-id ] [--cron ] --output json -uip or eval schedule pause --process-key --output json -uip or eval schedule resume --process-key --output json -uip or eval schedule delete --process-key --output json +uip eval schedule list --process-key --output json +uip eval schedule get --process-key --output json +uip eval schedule update --process-key [--eval-set-id ] [--cron ] --output json +uip eval schedule pause --process-key --output json +uip eval schedule resume --process-key --output json +uip eval schedule delete --process-key --output json ``` Output codes: `EvalScheduleList`, `EvalScheduleDetails`, `EvalScheduleUpdated`, `EvalSchedulePaused`, `EvalScheduleResumed`, `EvalScheduleDeleted`. @@ -397,7 +397,7 @@ Create evaluators, eval sets, and data points via CRUD, then run against the eva ```bash # 1. Create an evaluator -uip or eval evaluator create \ +uip eval evaluator create \ --process-key "$PROCESS_KEY" --workload-id "$WORKLOAD_ID" --folder-key "$FOLDER_KEY" \ --name "Semantic Similarity" --description "LLM output comparison" \ --evaluator-type-id uipath-llm-judge-output-semantic-similarity \ @@ -405,26 +405,26 @@ uip or eval evaluator create \ --output json # 2. Create an eval set linking the evaluator -uip or eval eval-set create \ +uip eval eval-set create \ --process-key "$PROCESS_KEY" --workload-id "$WORKLOAD_ID" --folder-key "$FOLDER_KEY" \ --name "Smoke Tests" --evaluator-refs "$EVALUATOR_ID" \ --output json # 3. Add data points to the eval set -uip or eval evaluation create \ +uip eval evaluation create \ --process-key "$PROCESS_KEY" --eval-set-id "$EVAL_SET_ID" --folder-key "$FOLDER_KEY" \ --name "Greeting test" --inputs '{"input":"hello"}' \ --expected-output '{"content":"Hi there!"}' \ --output json # 4. Update the eval set to add more evaluator refs if needed -uip or eval eval-set update "$EVAL_SET_ID" \ +uip eval eval-set update "$EVAL_SET_ID" \ --process-key "$PROCESS_KEY" \ --evaluator-refs "$EVALUATOR_ID" "$ANOTHER_EVALUATOR_ID" \ --output json # 5. Run the eval — items and evaluators are passed inline -uip or eval execute-and-evaluate \ +uip eval execute-and-evaluate \ --process-key "$PROCESS_KEY" \ --workload-id "$WORKLOAD_ID" \ --eval-set-id "$EVAL_SET_ID" \ @@ -433,11 +433,11 @@ uip or eval execute-and-evaluate \ --output json # 6. Check results -uip or eval run list --process-key "$PROCESS_KEY" --output json -uip or eval run results "$EVAL_SET_RUN_ID" --process-key "$PROCESS_KEY" --output json +uip eval run list --process-key "$PROCESS_KEY" --output json +uip eval run results "$EVAL_SET_RUN_ID" --process-key "$PROCESS_KEY" --output json # 7. Schedule recurring runs (workload-id and folder-key auto-resolved from eval set) -uip or eval schedule create \ +uip eval schedule create \ --process-key "$PROCESS_KEY" \ --eval-set-id "$EVAL_SET_ID" \ --cron "0 9 * * *" --output json diff --git a/tests/tasks/uipath-agents/lowcode/eval_run/eval_run.yaml b/tests/tasks/uipath-agents/lowcode/eval_run/eval_run.yaml index 48b8da56f1..87921e4701 100644 --- a/tests/tasks/uipath-agents/lowcode/eval_run/eval_run.yaml +++ b/tests/tasks/uipath-agents/lowcode/eval_run/eval_run.yaml @@ -18,7 +18,7 @@ description: > there, hand-writing UUIDs is a documented anti-pattern. Deliberately scoped to a SINGLE eval run. `uip agent eval run compare` needs a second run, and observed cloud-run duration varies ~4x - (393s-1476s). Compare and `uip or eval run-offline-evals` belong in their own + (393s-1476s). Compare and `uip eval run-offline-evals` belong in their own task. tags: [uipath-agents, e2e, mode:operate, lifecycle:setup, low-code, feature:eval]