diff --git a/.github/workflows/lint-ext-azure-ai-evaluations.yml b/.github/workflows/lint-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..2cb72ea1de4 --- /dev/null +++ b/.github/workflows/lint-ext-azure-ai-evaluations.yml @@ -0,0 +1,28 @@ +name: ext-azure-ai-evaluations-ci + +on: + pull_request: + paths: + - "cli/azd/extensions/azure.ai.evaluations/**" + - ".github/workflows/lint-ext-azure-ai-evaluations.yml" + - ".github/workflows/verify-ext-providers.yml" + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # required by reusable workflow lint-go.yml + +jobs: + lint: + uses: ./.github/workflows/lint-go.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations + + verify-providers: + uses: ./.github/workflows/verify-ext-providers.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations diff --git a/cli/azd/extensions/azure.ai.evaluations/.gitignore b/cli/azd/extensions/azure.ai.evaluations/.gitignore new file mode 100644 index 00000000000..0d5b6d76489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.gitignore @@ -0,0 +1,5 @@ +# Test report written by ci-test.ps1 for the pipeline to publish. +junitTestReport.xml + +# Debug log written when --debug or AZD_EXT_DEBUG is set. +azd-ai-eval-*.log diff --git a/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml new file mode 100644 index 00000000000..9777522d023 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml @@ -0,0 +1,21 @@ +version: "2" + +linters: + default: none + enable: + - gosec + - lll + - unused + - errorlint + settings: + lll: + line-length: 220 + tab-width: 4 + gosec: + excludes: + - G204 # Subprocess launched with variable (bicep build invoked in tests) + - G304 # Potential file inclusion via variable + +formatters: + enable: + - gofmt diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md new file mode 100644 index 00000000000..1a0e537e970 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -0,0 +1,29 @@ +# Release History + +## 1.0.14-beta (Unreleased) + +First release of the Foundry evaluations extension. + +### Features Added + +- Initial release of the Foundry evaluations extension, `azd ai eval`. +- `init` scaffolds `evals/azure.eval.yaml` next to an agent and adds the service + entry that `$ref`s it to `azure.yaml`, making no service calls. +- `generate` synthesizes a rubric and dataset from the agent's context, writes + them under `evals/`, and merges `source:` references into the deployment spec + while preserving comments, ordering and neighboring entries. +- `run` creates the eval group when it does not exist, starts a run, and + summarizes the result. +- `azure.ai.eval` service-target provider deploys datasets, evaluators and eval + groups during a deploy, reconciling them in dependency order. +- Change detection so a repeated deploy publishes no redundant versions: + datasets are fingerprinted locally, evaluator definitions are compared on the + keys the author wrote, and eval groups are recreated only when their own + declaration changes. +- Atomic commands for every operation: `dataset`, `evaluator`, `run` and + `run output` subcommands, all supporting `-o json` and `--no-prompt`. +- Testing criteria are shaped from each evaluator's published contract, so + evaluators requiring inputs beyond the agent shape — `ground_truth`, + `context`, `instruction_id_list` — work by binding them to dataset columns. + A required column the dataset does not carry is reported before the request + is sent, naming the column. diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md new file mode 100644 index 00000000000..148b7fbb711 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -0,0 +1,181 @@ +# Azure Developer CLI (azd) Evaluations Extension + +Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them +with `azd up`, and run them from the terminal. + +```bash +azd ai eval init # scaffold evals/ next to your agent +azd ai eval generate # synthesize a rubric and dataset from the agent +azd up # register datasets and evaluators, create the eval group +azd ai eval run # run the evaluation and summarize the results +``` + +## What gets deployed + +Eval resources are one service entry in `azure.yaml`, normally a `$ref` to a +file under `evals/`: + +```yaml +# azure.yaml +services: + ai-project: + host: azure.ai.project + evals: + host: azure.ai.eval + uses: [ai-project] + $ref: ./evals/azure.yaml +``` + +```yaml +# evals/azure.yaml +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evalGroups: + - name: support-quality + dataset: support-golden + evaluators: + - builtin.task_adherence + - support-quality + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano +``` + +`azd up` reconciles **datasets → evaluators → eval groups**, in that order, +because a group references the versions the first two resolve to. + +Relative paths inside a `$ref`'d file resolve against **that file's** +directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. + +### Repeated deploys do not create redundant versions + +Datasets are fingerprinted locally, because the dataset API exposes no content +hash and comparing against the service would mean downloading the blob on every +deploy. Evaluator definitions are compared against the service, but only on the +keys you authored — the service adds `data_schema`, `init_parameters` and +`metrics` of its own. + +Eval groups are immutable, so a change to a group's evaluators, target or +options creates a new group and a new id. The id is cached in the azd +environment so repeat runs stay comparable. + +## Commands + +| Group | Commands | +|---|---| +| `azd ai eval` | `init` · `generate` · `run` | +| `azd ai eval dataset` | `create` · `list` · `show` · `update` · `delete` | +| `azd ai eval evaluator` | `upload` · `list` · `show` · `update` · `delete` · `builtins` | +| `azd ai eval run` | `start` · `list` · `show` · `cancel` | +| `azd ai eval results` | `show` · `export` | + +`create` and `update` both publish a new immutable version; the server +auto-increments and nothing mutates in place. + +Every command supports `-o json` and `--no-prompt`, so the whole surface is +usable from CI. + +## Evaluators + +Built-ins need no declaration — reference them as `builtin.` and list +them with `azd ai eval evaluator builtins`. + +Evaluators do not share an input contract, so the CLI reads each one's +published contract and shapes the request to match. An evaluator needing an +input your dataset does not carry is reported before the request is sent, with +the column named, rather than as a service-side rejection. + +A custom rubric is a JSON list of weighted dimensions: + +```json +{ + "dimensions": [ + { "id": "accuracy", "description": "The answer is factually correct.", "weight": 5 }, + { "id": "tone", "description": "The answer is polite and professional.", "weight": 2 } + ] +} +``` + +`weight` is an **integer from 1 to 10**. Weights do not need to sum to +anything. + +## Choosing a project + +The project endpoint is resolved in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment, then + `AZURE_AI_PROJECT_ENDPOINT` there +3. `extensions.ai-agents.project.context.endpoint` in azd's global config, + which `azure.ai.agents` writes and this extension only reads +4. `FOUNDRY_PROJECT_ENDPOINT` in the host environment, then + `AZURE_AI_PROJECT_ENDPOINT` + +Level 3 is worth knowing about: it is machine-wide rather than per-project, so +a project context left behind by `azd ai agent` somewhere else takes precedence +over the variable exported in this shell. `--debug` prints which level answered. + +## Local development + +### Prerequisites + +- Go (the version in `go.mod`; `GOTOOLCHAIN=auto` fetches it) +- [azd](https://aka.ms/azd) and the extension developer kit: + `azd ext install microsoft.azd.extensions` + +### Build, test, install + +```bash +azd x build # compile and install into the local azd +azd x pack # package the artifacts +azd x publish # register in the local extension source +azd ext install azure.ai.evaluations --source local +``` + +```bash +go test ./internal/... # unit tests +``` + +### Live integration tests + +These talk to a real Foundry project, so they are excluded from the default +build by the `live` tag and additionally gated on an environment variable: + +```bash +export AZURE_AI_EVAL_E2E_LIVE=1 +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export AZURE_AI_EVAL_MODEL=gpt-4.1-nano # optional judge model +export AZURE_AI_EVAL_AGENT= # optional, enables the run phase + +go test -tags live ./internal/cmd/ ./tests/live/ +``` + +They clean up every resource they create. + +### Debug logging + +Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes +it to a dated log file rather than the terminal. + +## TODO before release + +Both are files the azd extensions team owns, so they are not changed here: + +- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.evaluations` + entry. Until it exists `azd extension install azure.ai.evaluations` cannot + resolve, so the extension is only reachable through `azd x pack` + + `azd x publish` into the local source registry. +- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.evaluations/`. + Every sibling Foundry extension has an entry; without one, PRs here get no + reviewer routing. +- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only + after the registry entry lands. Declaring a dependency that cannot resolve + breaks installing the bundle. diff --git a/cli/azd/extensions/azure.ai.evaluations/build.ps1 b/cli/azd/extensions/azure.ai.evaluations/build.ps1 new file mode 100644 index 00000000000..e016344f65b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.ps1 @@ -0,0 +1,78 @@ +# Ensure script fails on any error +$ErrorActionPreference = 'Stop' + +# Get the directory of the script +$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Change to the script directory +Set-Location -Path $EXTENSION_DIR + +# Create a safe version of EXTENSION_ID replacing dots with dashes +$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' + +# Define output directory +$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } + +# Create output directory if it doesn't exist +if (-not (Test-Path -Path $OUTPUT_DIR)) { + New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null +} + +# Get Git commit hash and build date +$COMMIT = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Failed to get git commit hash" + exit 1 +} +$BUILD_DATE = ((Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")) + +# List of OS and architecture combinations +if ($env:EXTENSION_PLATFORM) { + $PLATFORMS = @($env:EXTENSION_PLATFORM) +} +else { + $PLATFORMS = @( + "windows/amd64", + "windows/arm64", + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64" + ) +} + +$VERSION_PATH = "azureaieval/internal/version" + +# Loop through platforms and build +foreach ($PLATFORM in $PLATFORMS) { + $OS, $ARCH = $PLATFORM -split '/' + + $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" + + if ($OS -eq "windows") { + $OUTPUT_NAME += ".exe" + } + + Write-Host "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + if (Test-Path -Path $OUTPUT_NAME) { + Remove-Item -Path $OUTPUT_NAME -Force + } + + # Set environment variables for Go build + $env:GOOS = $OS + $env:GOARCH = $ARCH + + go build ` + -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` + -o $OUTPUT_NAME + + if ($LASTEXITCODE -ne 0) { + Write-Host "An error occurred while building for $OS/$ARCH" + exit 1 + } +} + +Write-Host "Build completed successfully!" +Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/build.sh b/cli/azd/extensions/azure.ai.evaluations/build.sh new file mode 100644 index 00000000000..4165a516ac4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Get the directory of the script +EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Change to the script directory +cd "$EXTENSION_DIR" || exit + +# Create a safe version of EXTENSION_ID replacing dots with dashes +EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" + +# Define output directory +OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" + +# Create output and target directories if they don't exist +mkdir -p "$OUTPUT_DIR" + +# Get Git commit hash and build date +COMMIT=$(git rev-parse HEAD) +BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# List of OS and architecture combinations +if [ -n "$EXTENSION_PLATFORM" ]; then + PLATFORMS=("$EXTENSION_PLATFORM") +else + PLATFORMS=( + "windows/amd64" + "windows/arm64" + "darwin/amd64" + "darwin/arm64" + "linux/amd64" + "linux/arm64" + ) +fi + +VERSION_PATH="azureaieval/internal/version" + +# Loop through platforms and build +for PLATFORM in "${PLATFORMS[@]}"; do + OS=$(echo "$PLATFORM" | cut -d'/' -f1) + ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) + + OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" + + if [ "$OS" = "windows" ]; then + OUTPUT_NAME+='.exe' + fi + + echo "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" + + # Set environment variables for Go build + GOOS=$OS GOARCH=$ARCH go build \ + -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ + -o "$OUTPUT_NAME" + + if [ $? -ne 0 ]; then + echo "An error occurred while building for $OS/$ARCH" + exit 1 + fi +done + +echo "Build completed successfully!" +echo "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 new file mode 100644 index 00000000000..403bc23b08d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 @@ -0,0 +1,114 @@ +param( + [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), + [string] $SourceVersion = (git rev-parse HEAD), + [switch] $CodeCoverageEnabled, + # Accepted because the shared CI template always passes it. This extension + # has no record/playback mode, so there is no second binary to produce. + [switch] $BuildRecordMode, + [string] $MSYS2Shell, # path to msys2_shell.cmd + [string] $OutputFileName +) +$PSNativeCommandArgumentPassing = 'Legacy' + +# Remove any previously built binaries. +go clean + +if ($LASTEXITCODE) { + Write-Host "Error running go clean" + exit $LASTEXITCODE +} + +# Run `go help build` for detail on these flags. +$buildFlags = @( + # Remove file system paths from the binary. Recorded file names become a + # module path@version, or a plain import path for the standard library. + "-trimpath", + + # Position Independent Executable, for memory-corruption hardening across + # platforms. On Windows this enables ASLR and sets DYNAMICBASE and + # HIGH-ENTROPY-VA in the PE header. + "-buildmode=pie" +) + +if ($CodeCoverageEnabled) { + $buildFlags += "-cover" +} + +# cfi: Control Flow Integrity, cfg: Control Flow Guard, +# osusergo: use the pure Go user lookup. +$tagsFlag = "-tags=cfi,cfg,osusergo" + +# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. +$ldFlag = "-ldflags=-s -w " + + "-X 'azureaieval/internal/version.Version=$Version' " + + "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + + "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " + +if ($IsWindows) { + Write-Host "Building for Windows" +} +elseif ($IsLinux) { + Write-Host "Building for linux" + + # Disable cgo for the x64 Linux build. This also links statically, which + # widens compatibility with older Linux distributions. + if ($env:GOARCH -ne "arm64") { + $env:CGO_ENABLED = "0" + } +} +elseif ($IsMacOS) { + Write-Host "Building for macOS" +} + +$outputFlag = "-o=$OutputFileName" + +$buildFlags += @( + $tagsFlag, + $ldFlag, + $outputFlag +) + +function PrintFlags() { + param( + [string] $flags + ) + + # Format the flags so they can be pasted straight into pwsh. + $i = 0 + foreach ($buildFlag in $buildFlags) { + # Quote values so characters such as ',' survive a repaste. Not needed + # for the direct invocation below. + $argWithValue = $buildFlag.Split('=', 2) + if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { + $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" + } + + if ($i -eq $buildFlags.Length - 1) { + Write-Host " $buildFlag" + } + else { + Write-Host " $buildFlag ``" + } + $i++ + } +} + +$oldGOEXPERIMENT = $env:GOEXPERIMENT +# Opt into per-iteration loop variables, which is what most readers expect and +# what the Go team intends to make the default. +$env:GOEXPERIMENT = "loopvar" + +try { + Write-Host "Running: go build ``" + PrintFlags -flags $buildFlags + go build @buildFlags + if ($LASTEXITCODE) { + Write-Host "Error running go build" + exit $LASTEXITCODE + } + + Write-Host "go build succeeded" +} +finally { + $env:GOEXPERIMENT = $oldGOEXPERIMENT +} diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 new file mode 100644 index 00000000000..5415bd107a8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -0,0 +1,61 @@ +# Runs the unit tests and writes a JUnit report. +# +# The pipeline publishes **/junitTestReport.xml from the extension directory, +# so the report has to be written under that name for results to show up in the +# build. gotestsum produces it; the go test fallback does not, so the fallback +# only runs when gotestsum is unavailable. +# +# The live integration tests are excluded: they carry the `live` build tag, so +# an untagged run does not compile them, and they additionally require +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked +# below, so a change that breaks them cannot reach main unnoticed. +# +# TODO before the first release: PR CI runs this script on windows, linux and +# darwin amd64, so the untagged tests are covered on all three. The live and +# hero suites are only type-checked, never executed, and both have only ever +# run on Windows by hand. Run them once on linux, where they assume a path +# separator and shell out to `azd` and to a proxy address. + +$gopath = go env GOPATH +$gotestsumBinary = "gotestsum" +# $IsWindows only exists on PowerShell 6 and later. On Windows PowerShell 5.1 it +# is undefined, so the suffix was never appended, the binary was never found, +# and the run silently fell back to `go test` with no JUnit report. +if ($env:OS -eq "Windows_NT") { + $gotestsumBinary += ".exe" +} +# Windows PowerShell 5.1 takes a single child path, so the three-argument form +# fails outright there. Nesting is what every version accepts. +$gotestsum = Join-Path (Join-Path $gopath "bin") $gotestsumBinary + +Write-Host "Running unit tests..." + +if (Test-Path $gotestsum) { + & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 +} else { + Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow + go test ./... -v -count=1 +} + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +# The tagged suites are never run here, so without this nothing compiles them +# and a change that breaks one reaches main silently. Type-checking needs no +# credentials, so it costs a few seconds and runs everywhere the tests do. +Write-Host "" +Write-Host "Type-checking the live and hero suites..." +go vet -tags live,hero ./... + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "All tests passed!" -ForegroundColor Green +exit 0 diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml new file mode 100644 index 00000000000..96751b9f9a0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -0,0 +1,44 @@ +import: ../../.vscode/cspell.yaml +words: + # Go module and package names + - azureaieval + - evalcore + - exterrors + - httptest + - projectctx + - urlsafe + - creack + # Service identifiers and API fields + - evalrun + - lookback + - AOAI + - ARMID + # Built-in evaluator names + - ifeval + - groundedness + # Repository names + - foundrysdk + # Version-control systems a read-only checkout can come from + - TFVC + # GitHub metadata files named in the README checklist + - CODEOWNERS + # Possessive of an acronym cspell does not inflect on its own + - CLI's + # Deliberate misspellings: the fixtures the unknown-key tests are about + - evaulators + - verison + # Terms + - inlines + - negotiables + - parseable + - preselection + - retargeted + - subsetting + - undeployed + - undoable + - unbuildable + - unorderable + - unpassed + - unjudged + - unscored + - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml new file mode 100644 index 00000000000..a192d331e23 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +# NOTE: Make sure version.txt is in sync with this version. +version: 1.0.14-beta +requiredAzdVersion: ">=1.27.1" +language: go +capabilities: + - custom-commands + - service-target-provider + - metadata +providers: + - name: azure.ai.eval + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +examples: + - name: init + description: Scaffold evaluation config for an agent. + usage: azd ai eval init + - name: run + description: Run an evaluation and summarize the results. + usage: azd ai eval run diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod new file mode 100644 index 00000000000..50ad86238b1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -0,0 +1,106 @@ +module azureaieval + +go 1.26.4 + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/fatih/color v1.18.0 + github.com/gofrs/flock v0.12.1 + github.com/google/uuid v1.6.0 + github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 + go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect + github.com/alecthomas/chroma/v2 v2.20.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/braydonk/yaml v0.9.0 // indirect + github.com/buger/goterm v1.0.4 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/glamour v0.10.0 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.10.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cli/browser v1.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/drone/envsubst v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golobby/container/v3 v3.3.2 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jmespath-community/go-jmespath v1.1.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.41.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect + github.com/microsoft/go-deviceid v1.0.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/theckman/yacspin v0.13.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum new file mode 100644 index 00000000000..6c14ec16af1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -0,0 +1,318 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= +github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= +github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= +github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= +github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= +github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= +github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= +github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= +github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= +github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= +github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= +github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= +github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= +github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= +github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= +github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go new file mode 100644 index 00000000000..47f36fc7cce --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation spec names an instructions file relative to itself, not to the +// working directory, so `generate --config ` reads the same file the +// author sees next to the spec. +func TestDeclaredInstructions_ResolvesRelativeToTheSpec(t *testing.T) { + dir := t.TempDir() + specDir := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o750)) + + body := "Answer only from the product catalog." + require.NoError(t, os.WriteFile( + filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) + + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(specDir, "generate.yaml")) + require.NoError(t, err) + assert.Equal(t, body, got, "the file's contents should be used, trimmed") +} + +// A path can be declared before that file exists. Treating the gap as an error +// would break the flow `init` itself scaffolds. +func TestDeclaredInstructions_MissingFileIsNotAnError(t *testing.T) { + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(t.TempDir(), "generate.yaml")) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestDeclaredInstructions_UnsetIsEmpty(t *testing.T) { + got, err := declaredInstructions("", "generate.yaml") + require.NoError(t, err) + assert.Empty(t, got) +} + +// Only the newest version is read, and an agent with no published version must +// not panic the caller. +func TestAgentInstructions(t *testing.T) { + var agent eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "support", + "versions": { "latest": { "version": "2", "definition": { + "model": "gpt-5-mini", + "instructions": " You are a support assistant.\n" } } } + }`), &agent)) + assert.Equal(t, "You are a support assistant.", agent.Instructions()) + + var empty eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{"name":"x","versions":{}}`), &empty)) + assert.Empty(t, empty.Instructions(), "an agent with no published version has no instructions") + + var nilAgent *eval_api.Agent + assert.Empty(t, nilAgent.Instructions()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go new file mode 100644 index 00000000000..0c3ddb78734 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +// API versions used by the Foundry data plane. +const ( + // ProjectEndpointAPIVersion covers datasets, evaluators, and evaluator + // generation jobs on the project endpoint. + ProjectEndpointAPIVersion = "2025-11-15-preview" + + // DataGenerationAPIVersion covers dataset generation jobs. + DataGenerationAPIVersion = "v1" + + // OpenAI-compatible eval and run calls send no api-version, so there + // is deliberately no constant for them. +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go new file mode 100644 index 00000000000..f120a2cdfb3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Every verb that takes a refuses an invalid one locally. +// +// The guard was written for the dataset verbs and never reached the evaluator +// ones, so `azd ai eval dataset show "my set"` explained itself and +// `azd ai eval evaluator show "my rubric"` returned the service's 400 wrapped +// in four levels of JSON. Walking the tree rather than listing the verbs is +// the point: a verb added later is covered without anyone remembering to. +func TestEveryNamedAssetVerbRefusesAnInvalidName(t *testing.T) { + const badName = "has space" + + var walk func(cmd *cobra.Command, path string) + checked := 0 + + walk = func(cmd *cobra.Command, path string) { + for _, sub := range cmd.Commands() { + walk(sub, strings.TrimSpace(path+" "+sub.Name())) + } + if cmd.RunE == nil || !strings.Contains(cmd.Use, "") { + return + } + + checked++ + // The guard has to come first: it runs before the client is built, so + // a mistyped name costs neither a round trip nor an azd connection. + err := cmd.RunE(cmd, []string{badName}) + require.Errorf(t, err, "%s accepted %q", path, badName) + assert.Containsf(t, err.Error(), "is invalid", + "%s refused %q, but not by naming the name", path, badName) + } + + walk(newDatasetCommand(), "dataset") + walk(newEvaluatorCommand(), "evaluator") + + assert.GreaterOrEqual(t, checked, 8, + "both command groups take a name on create, update, show, delete and versions list") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go new file mode 100644 index 00000000000..0844cf55a43 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -0,0 +1,440 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" +) + +// evaluatorSchemas indexes the published contract of every evaluator a group +// can reference. +// +// Built-ins have to be asked for separately. An unfiltered list returns only +// the project's own evaluators, so relying on it leaves every built-in without +// a schema and falling back to legacyInputs — which happens to match +// query/response and so looks right for the common evaluators while quietly +// dropping the fields anything else needs. +// +// A failure is deliberately not fatal: without schemas the builder falls back +// to the agent-target shape, which is what it always used to send. +func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { + if ec.schemas != nil { + return ec.schemas + } + + index := map[string]*eval_api.EvaluatorSummary{} + complete := true + for _, filter := range []string{"", eval_api.EvaluatorTypeBuiltin} { + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + complete = false + continue + } + maps.Copy(index, list.ByName()) + } + if len(index) == 0 { + return nil + } + // Only a complete read is worth keeping. Caching a half of it would leave + // every later eval validating against legacyInputs, which accepts fields + // the evaluator never declared. + if complete { + ec.schemas = index + } + return index +} + +// sampleBindings are the fields an agent target produces at run time. Anything +// an evaluator accepts that is not in this set has to come from a dataset +// column instead. +var sampleBindings = map[string]string{ + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", +} + +// sampleBindingsFor returns the run-time bindings a target of this kind can +// satisfy. An empty target kind means nothing is invoked, so nothing is bound. +// +// A model target gets none: a model answers as plain text and calls no tools, +// so binding an agent's richer output would leave the evaluator waiting on +// fields the run never produces. +func sampleBindingsFor(targetType string) map[string]string { + if targetType == project.TargetTypeAgent { + return sampleBindings + } + return nil +} + +// legacyInputs is the mapping used when the service publishes no schema for an +// evaluator, which is the case for freshly uploaded custom evaluators. It +// matches the agent-target shape. +var legacyInputs = []string{"query", "response", "tool_calls", "tool_definitions"} + +// criterionPlan is the resolved binding for one evaluator. +type criterionPlan struct { + dataMapping map[string]string + initParams map[string]any + // itemFields are the fields sourced from dataset columns; they have to be + // declared in the item schema. + itemFields []string +} + +// conversationField carries a whole conversation. The service rejects a +// mapping that pairs it with the turn-level fields: +// +// Evaluator 'builtin.task_completion' has both 'messages' and +// 'query'/'response' in data_mapping. Use 'messages' for conversation-level +// evaluation or 'query'/'response' for turn-level evaluation, but not both. +const conversationField = "messages" + +// turnFields are the per-turn counterparts to conversationField. +var turnFields = []string{"query", "response"} + +// selectLevelFields resolves the conversation/turn exclusivity for evaluators +// that accept both shapes, keeping whichever matches the evaluation level. +// Required fields are never dropped, so a genuine conflict still surfaces as a +// missing-field error rather than being silently reshaped. +func selectLevelFields(accepted, required []string, level string) []string { + isRequired := make(map[string]bool, len(required)) + for _, name := range required { + isRequired[name] = true + } + + acceptsConversation := false + acceptsTurn := false + for _, field := range accepted { + if field == conversationField { + acceptsConversation = true + } + for _, turn := range turnFields { + if field == turn { + acceptsTurn = true + } + } + } + if !acceptsConversation || !acceptsTurn { + return accepted + } + + drop := map[string]bool{} + if strings.EqualFold(level, project.EvaluationLevelConversation) { + for _, turn := range turnFields { + drop[turn] = true + } + } else { + drop[conversationField] = true + } + + kept := make([]string, 0, len(accepted)) + for _, field := range accepted { + if drop[field] && !isRequired[field] { + continue + } + kept = append(kept, field) + } + return kept +} + +// planCriterion shapes one evaluator's bindings from its published contract. +// +// Evaluators do not share an input contract: builtin.similarity needs +// ground_truth, builtin.retrieval needs context, and builtin.ifeval needs +// instruction_id_list. Sending one fixed mapping to all of them earns a +// service-side MissingRequiredDataMapping rejection, so the mapping is derived +// per evaluator and anything unsatisfiable is reported before the request is +// sent. +func planCriterion( + ref evalcore.EvaluatorRef, + schema *eval_api.EvaluatorSummary, + targetBindings map[string]string, + datasetColumns map[string]bool, + level string, +) (*criterionPlan, error) { + accepted := legacyInputs + var required []string + // A published schema is authoritative even when it is empty: an empty + // property set means the evaluator accepts nothing, which is different from + // publishing no schema at all. + if dataSchema := schema.DataSchema(); dataSchema != nil { + accepted = dataSchema.PropertyNames() + required = dataSchema.Required + } + accepted = selectLevelFields(accepted, required, level) + + plan := &criterionPlan{ + dataMapping: map[string]string{}, + initParams: map[string]any{}, + } + + for _, field := range accepted { + if binding, ok := targetBindings[field]; ok { + plan.dataMapping[field] = binding + continue + } + // Everything else comes from the dataset. When the columns are known, + // bind only the ones that exist so optional fields stay unbound rather + // than resolving to nothing at run time. + if datasetColumns != nil && !datasetColumns[field] { + continue + } + plan.dataMapping[field] = fmt.Sprintf("{{item.%s}}", field) + plan.itemFields = append(plan.itemFields, field) + } + + // A declared mapping is the author saying the inference got it wrong, so it + // wins. Anything it binds to an item column is a column the schema has to + // declare, whether or not inference found it. + for field, binding := range ref.DataMapping { + plan.dataMapping[field] = binding + if column, ok := itemColumn(binding); ok && !contains(plan.itemFields, column) { + plan.itemFields = append(plan.itemFields, column) + } + } + + var missing []string + for _, field := range required { + if _, ok := plan.dataMapping[field]; !ok { + missing = append(missing, field) + } + } + if len(missing) > 0 { + return nil, messages.EvaluatorNeedsFields(ref.Evaluator, missing) + } + + if !schema.SupportsLevel(level) { + return nil, messages.EvaluatorLevelUnsupported( + ref.Evaluator, level, schema.SupportedEvaluationLevels) + } + + initSchema := schema.InitSchema() + accepts := func(name string) bool { + // Only an absent schema falls back to the historical parameters; + // builtin.ifeval publishes an empty one and takes none. + if initSchema == nil { + return name == "deployment_name" || name == "threshold" + } + return initSchema.Accepts(name) + } + + // Evaluators disagree on what the judge model is called: built-ins declare + // deployment_name, custom rubrics declare model. The declaration names one + // of them; bind whichever the evaluator actually accepts rather than + // forwarding a spelling it will reject. + for name, value := range ref.InitializationParameters { + if accepts(name) { + plan.initParams[name] = value + continue + } + if alias, ok := judgeModelAliases[name]; ok && accepts(alias) { + plan.initParams[alias] = value + } + } + if level != "" && accepts("evaluation_level") { + plan.initParams["evaluation_level"] = level + } + + if initSchema != nil { + var missingInit []string + for _, name := range initSchema.Required { + if _, ok := plan.initParams[name]; !ok { + missingInit = append(missingInit, name) + } + } + if len(missingInit) > 0 { + return nil, messages.EvaluatorNeedsInitParams(ref.Evaluator, missingInit) + } + } + + return plan, nil +} + +// checkEvaluatorRequirements refuses a declaration the evaluators cannot +// satisfy, before anything is published. +// +// The same checks happen while building the request, but that runs after the +// datasets and evaluators have been pushed -- so a missing judge deployment +// cost an immutable dataset version per attempt, and the version numbers climb +// whether or not the eval was ever created. Only what the published contract +// alone can settle is checked here: the data mapping needs the dataset's +// columns, which is a separate question. +func checkEvaluatorRequirements( + eval *project.Eval, + schemas map[string]*eval_api.EvaluatorSummary, +) error { + level := resolveLevel(eval) + for _, ref := range eval.Evaluators { + schema := schemas[ref.Evaluator] + if schema == nil { + // Nothing published to check against. The service still gets the + // last word, which is what happened before this existed. + continue + } + if !schema.SupportsLevel(level) { + return messages.EvaluatorLevelUnsupported( + ref.Evaluator, level, schema.SupportedEvaluationLevels) + } + + initSchema := schema.InitSchema() + if initSchema == nil { + continue + } + var missing []string + for _, name := range initSchema.Required { + if declaredInitParam(ref, name) { + continue + } + if name == "evaluation_level" && level != "" { + continue + } + missing = append(missing, name) + } + if len(missing) > 0 { + return messages.EvaluatorNeedsInitParams(ref.Evaluator, missing) + } + } + return nil +} + +// declaredInitParam reports whether the reference supplies a parameter under +// either spelling of the judge deployment. +func declaredInitParam(ref evalcore.EvaluatorRef, name string) bool { + if _, ok := ref.InitializationParameters[name]; ok { + return true + } + if alias, ok := judgeModelAliases[name]; ok { + _, declared := ref.InitializationParameters[alias] + return declared + } + return false +} + +// judgeModelAliases maps the two spellings of the judge deployment onto each +// other, so one declaration works whichever the evaluator publishes. +var judgeModelAliases = map[string]string{ + "deployment_name": "model", + "model": "deployment_name"} + +// itemColumn reads the dataset column out of an `{{item.}}` binding. +func itemColumn(binding string) (string, bool) { + const prefix, suffix = "{{item.", "}}" + if !strings.HasPrefix(binding, prefix) || !strings.HasSuffix(binding, suffix) { + return "", false + } + name := strings.TrimSuffix(strings.TrimPrefix(binding, prefix), suffix) + if name == "" { + return "", false + } + return name, true +} + +func contains(values []string, want string) bool { + return slices.Contains(values, want) +} + +// buildEvalRequest converts an eval declaration into the create +// request. Each evaluator becomes a testing criterion bound to its own +// contract, and the item schema declares every dataset column those bindings +// reference. +// +// schemas may be nil or partial; an evaluator with no published contract falls +// back to the agent-target shape. datasetColumns may be nil, meaning the +// columns are unknown and every accepted field is assumed present. +func buildEvalRequest( + group *project.Eval, + schemas map[string]*eval_api.EvaluatorSummary, + datasetColumns map[string]bool, +) (*eval_api.CreateOpenAIEvalRequest, error) { + metadata := map[string]string{} + hasTarget := group.Target != nil && group.Target.Name != "" + targetType := "" + if hasTarget { + metadata[metaAgent] = group.Target.Name + targetType = group.Target.Type + if targetType == "" { + targetType = project.TargetTypeAgent + } + } + targetBindings := sampleBindingsFor(targetType) + metadata[metaEvalName] = group.Name + // The create request has no description field, so the group's own + // description rides in metadata rather than being dropped. + if group.Description != "" { + metadata[metaDescription] = group.Description + } + + level := group.EvaluationLevel + + req := &eval_api.CreateOpenAIEvalRequest{ + Name: group.Name, + Metadata: metadata, + } + + itemFields := map[string]bool{} + + for _, ref := range group.Evaluators { + schema := schemas[ref.Evaluator] + if schema == nil { + schema = &eval_api.EvaluatorSummary{Name: ref.Evaluator} + } + + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) + if err != nil { + return nil, err + } + + criterion := eval_api.TestingCriterion{ + Type: "azure_ai_evaluator", + // Name labels the criterion in results and defaults to the + // evaluator without its builtin prefix; EvaluatorName keeps it. + Name: ref.CriterionName(), + EvaluatorName: ref.Evaluator, + DataMapping: plan.dataMapping, + } + if ref.Version != "" { + criterion.EvaluatorVersion = ref.Version + } + if len(plan.initParams) > 0 { + criterion.InitializationParameters = plan.initParams + } + for _, field := range plan.itemFields { + itemFields[field] = true + } + + req.TestingCriteria = append(req.TestingCriteria, criterion) + } + + req.DataSourceConfig = &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: hasTarget, + ItemSchema: itemSchema(itemFields), + } + + return req, nil +} + +// itemSchema declares the dataset columns the criteria bind to. It always +// declares at least `query`, the column an agent target reads. +func itemSchema(fields map[string]bool) map[string]any { + if len(fields) == 0 { + fields = map[string]bool{"query": true} + } + properties := map[string]any{} + for field := range fields { + properties[field] = map[string]any{"type": "string"} + } + return map[string]any{ + "type": "object", + "properties": properties, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go new file mode 100644 index 00000000000..59e95d41c99 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the request buildEvalRequest produces is accepted by +// the real service. It lives in the cmd package on purpose: the tests under +// tests/live can only hand-roll a request, which validates the API but not the +// code that ships. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveBuild +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + }) + return sharedCred, sharedCredErr +} + +// credentialFlake is what a token refresh that overran its budget looks like +// by the time it reaches a test. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// retryingCredential retries a token request that failed for that reason. +// +// The refresh shells out to azd, and the SDK gives that subprocess ten +// seconds. On a machine already running the rest of this suite it sometimes +// does not finish in ten, and the failure lands on whichever test asked for a +// token at the wrong moment — reproducibly at 10.1s, and never when that test +// is run on its own. Retrying is right because nothing about the request was +// wrong: the same call succeeds moments later. +type retryingCredential struct { + inner azcore.TokenCredential +} + +func (c retryingCredential) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + var token azcore.AccessToken + var err error + for attempt := range 4 { + if attempt > 0 { + select { + case <-ctx.Done(): + return azcore.AccessToken{}, ctx.Err() + case <-time.After(time.Duration(attempt) * 2 * time.Second): + } + } + token, err = c.inner.GetToken(ctx, opts) + if err == nil || !strings.Contains(err.Error(), credentialFlake) { + return token, err + } + } + return token, err +} + +func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + cred, err := liveCredential() + require.NoError(t, err) + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = "gpt-4.1-nano" + } + return eval_api.NewEvalClient(endpoint, retryingCredential{inner: cred}), judge +} + +// TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project +// exposes, builds a group with the shipping builder, and posts it. +// +// Each evaluator declares a different input contract, so this is the test that +// would have caught the fixed data mapping: it previously produced a +// MissingRequiredDataMapping rejection for builtin.ifeval and would do so +// again for any evaluator whose contract the builder stops honouring. +func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, listed.Value) + + // Deliberately the production lookup rather than the listing above. Taking + // the schemas straight from a filtered list is what let this test pass + // while the shipping path resolved none of them: it built the input the + // product was failing to build. + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + for _, summary := range listed.Value { + summary := summary + t.Run(summary.Name, func(t *testing.T) { + require.NotNil(t, schemas[summary.Name], + "the shipping lookup did not resolve %s", summary.Name) + // Give the builder a dataset carrying every column the evaluator + // accepts, so a rejection means the request shape is wrong rather + // than the data being genuinely absent. + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, name := range ds.PropertyNames() { + columns[name] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + group := &project.Eval{ + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Evaluator: summary.Name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + EvaluationLevel: level, + } + + req, err := buildEvalRequest(group, schemas, columns) + require.NoError(t, err, "the builder must satisfy every published contract") + + created, err := client.CreateOpenAIEval(ctx, req) + require.NoError(t, err, + "the service rejected the request this extension builds for %s", summary.Name) + require.NotEmpty(t, created.ID) + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("%s accepted as %s", summary.Name, created.ID) + }) + } +} + +// TestLiveBuildRejectsMissingColumnsLocally proves the pre-flight check fires +// before the network call, so a user sees which column is missing instead of a +// service error naming an internal field path. +func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + schemas := listed.ByName() + + target, ok := schemas["builtin.ifeval"] + if !ok { + t.Skip("builtin.ifeval is not available in this project") + } + require.NotNil(t, target.DataSchema()) + require.NotEmpty(t, target.DataSchema().Required, + "this test relies on ifeval declaring required inputs") + + group := &project.Eval{ + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: "builtin.ifeval", + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + } + + // A dataset with only `query` cannot satisfy ifeval. + _, err = buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + t.Logf("pre-flight error: %v", err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go new file mode 100644 index 00000000000..0bacd287edf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// schema builds an evaluator contract the way the service publishes one. +func schema(name string, dataRequired, dataProps, initRequired, initProps []string, levels ...string) *eval_api.EvaluatorSummary { + toProps := func(names []string) map[string]any { + if names == nil { + return nil + } + out := map[string]any{} + for _, n := range names { + out[n] = map[string]any{"type": "string"} + } + return out + } + return &eval_api.EvaluatorSummary{ + Name: name, + SupportedEvaluationLevels: levels, + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{Required: dataRequired, Properties: toProps(dataProps)}, + InitParameters: &eval_api.JSONSchema{Required: initRequired, Properties: toProps(initProps)}, + }, + } +} + +func groupWith(evaluators []evalcore.EvaluatorRef, level string) *project.Eval { + return &project.Eval{ + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + EvaluationLevel: level, + } +} + +// withJudge declares the judge deployment where the service reads it from: an +// evaluator's initialization parameters, not a setting on the eval. It merges, +// so a parameter the reference already carries survives. +func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { + for i := range refs { + if refs[i].InitializationParameters == nil { + refs[i].InitializationParameters = map[string]any{} + } + refs[i].InitializationParameters["deployment_name"] = model + } + return refs +} + +// An agent evaluator takes its response from the sample and its query from the +// dataset. +func TestBuildBindsAgentFieldsFromSample(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_adherence": schema("builtin.task_adherence", + nil, []string{"query", "response", "tool_definitions", "messages"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold", "evaluation_level"}, + "turn"), + } + group := groupWith( + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Evaluator: "builtin.task_adherence"}), + "", + ) + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + // `messages` is not a dataset column here, so it stays unbound. + require.NotContains(t, mapping, "messages") +} + +// A required field the dataset does not carry is reported before the request +// is sent, naming the field. +func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + require.Contains(t, err.Error(), "instruction_kwargs") +} + +// The same evaluator succeeds once the dataset supplies the columns. +func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "instruction_id_list": true, + "instruction_kwargs": true, + }) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + // response is satisfied by the agent target. + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{item.instruction_id_list}}", mapping["instruction_id_list"]) + + // The item schema has to declare the columns the criteria reference. + props := req.DataSourceConfig.ItemSchema["properties"].(map[string]any) + require.Contains(t, props, "instruction_id_list") + require.Contains(t, props, "instruction_kwargs") +} + +// Initialization parameters are filtered to what the evaluator accepts. +// builtin.ifeval takes none, so nothing is sent even when a model is set. +func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { + threshold := 4.0 + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + nil, []string{"response"}, nil, nil, "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response", "ground_truth"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Evaluator: "builtin.ifeval", + InitializationParameters: map[string]any{"threshold": threshold}}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity", + InitializationParameters: map[string]any{"threshold": threshold}}, + ), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "query": true, "ground_truth": true, + }) + require.NoError(t, err) + + // ifeval accepts no init parameters at all. + require.Empty(t, req.TestingCriteria[0].InitializationParameters) + + // similarity accepts both, and never the `model` alias. + params := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", params["deployment_name"]) + require.InDelta(t, 4.0, params["threshold"], 0.0001) + require.NotContains(t, params, "model") +} + +// evaluation_level is an initialization parameter, not run metadata, and only +// on evaluators that declare it. +func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith(withJudge("m", + evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + ), "turn") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) + require.NotContains(t, req.TestingCriteria[1].InitializationParameters, "evaluation_level") +} + +// An evaluator that does not support the requested level is rejected with the +// levels it does support. +func TestBuildRejectsUnsupportedLevel(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), + "conversation") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "conversation") + require.Contains(t, err.Error(), "turn") +} + +// A required init parameter with no judge model configured is caught locally. +func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "deployment_name") +} + +// An evaluator with no published contract keeps the historical agent-target +// shape, so custom evaluators still deploy. +func TestBuildFallsBackWithoutSchema(t *testing.T) { + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "my-custom-evaluator"}), "") + + req, err := buildEvalRequest(group, nil, nil) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_calls}}", mapping["tool_calls"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + require.Equal(t, "m", req.TestingCriteria[0].InitializationParameters["deployment_name"]) +} + +// `messages` and `query`/`response` are mutually exclusive; the evaluation +// level picks which shape is bound. Sending both is rejected by the service. +func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response", "messages", "tool_definitions"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + } + columns := map[string]bool{"query": true, "messages": true, "response": true} + + // Turn level keeps query/response and drops messages. + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "turn") + req, err := buildEvalRequest(turn, schemas, columns) + require.NoError(t, err) + mapping := req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "query") + require.NotContains(t, mapping, "messages") + + // Conversation level keeps messages and drops query/response. + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "conversation") + req, err = buildEvalRequest(conv, schemas, columns) + require.NoError(t, err) + mapping = req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "messages") + require.NotContains(t, mapping, "query") + require.NotContains(t, mapping, "response") + + // An unset level behaves as turn, matching the service default. + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), "") + req, err = buildEvalRequest(dflt, schemas, columns) + require.NoError(t, err) + require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") +} + +// Evaluators disagree on what the judge model is called. Built-ins declare +// deployment_name; a custom rubric declares model, and rejects the eval with +// "requires model" if only deployment_name is sent. One declaration binds +// whichever the evaluator actually accepts. +func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + "my-rubric": schema("my-rubric", + nil, []string{"query", "response"}, + []string{"model"}, []string{"model"}, "turn"), + } + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + evalcore.EvaluatorRef{Evaluator: "my-rubric"}, + ), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + builtin := req.TestingCriteria[0].InitializationParameters + require.Equal(t, "gpt-4.1-nano", builtin["deployment_name"]) + require.NotContains(t, builtin, "model") + + custom := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", custom["model"]) + require.NotContains(t, custom, "deployment_name") +} + +// Without an agent target the sample bindings are unavailable, so every field +// has to come from the dataset and the sample schema is not requested. +func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + []string{"query", "response", "ground_truth"}, + []string{"query", "response", "ground_truth"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") + group.Target = nil + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "query": true, "response": true, "ground_truth": true, + }) + require.NoError(t, err) + require.False(t, req.DataSourceConfig.IncludeSampleSchema) + require.Equal(t, "{{item.response}}", req.TestingCriteria[0].DataMapping["response"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go new file mode 100644 index 00000000000..e92a76321f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation writes the artifact and then names it in the configuration, so +// what it produced is referenceable without a hand edit. Only the catalogs are +// touched: which evals use the artifact is the author's decision, and `init` is +// the command that makes it. + +// addDatasetToCatalog records a generated dataset in `datasets:`. +func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, "dataset", ref, func(cfg *project.EvalConfig) bool { + for i := range cfg.Datasets { + if cfg.Datasets[i].Name == ref.Name { + // Regeneration overwrites the file in place, so the entry only + // changes when the artifact moved. + if cfg.Datasets[i].Source == ref.Source { + return false + } + cfg.Datasets[i].Source = ref.Source + return true + } + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// addEvaluatorToCatalog records a generated evaluator in `evaluators:`. +func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, "evaluator", ref, func(cfg *project.EvalConfig) bool { + for i := range cfg.Evaluators { + if cfg.Evaluators[i].Name == ref.Name { + if cfg.Evaluators[i].Source == ref.Source { + return false + } + cfg.Evaluators[i].Source = ref.Source + return true + } + } + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// updateCatalog applies a change to the configuration and writes it back. +// +// A missing configuration is created holding only the catalog. `generate` runs +// before `init` on the golden path, and a downloaded artifact nobody recorded +// is the one state that goes stale. The file it creates has no evals and no +// azure.yaml entry, so it stays inert until init wires one. +func updateCatalog( + cmd *cobra.Command, + evalDir string, + kind string, + ref *project.ArtifactRef, + apply func(*project.EvalConfig) bool, +) error { + // Held across the read and the write: two generates adding different + // entries would otherwise both read the same state, and the second write + // would drop the first one's entry while reporting success. + unlock, err := project.LockEvalConfig(cmd.Context(), evalDir) + if err != nil { + return err + } + defer unlock() + + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return err + } + created := cfg == nil + if created { + cfg = &project.EvalConfig{} + } + if !apply(cfg) { + return nil + } + + if err := project.SaveEvalConfig(evalDir, cfg); err != nil { + return err + } + if !isJSON(cmd) { + // Resolved, not the current name: SaveEvalConfig writes back over a + // legacy file when that is what the project has, and the line has to + // name the file it actually wrote. + resolved, err := project.ResolveEvalConfigPath(evalDir) + if err != nil { + return err + } + path := filepath.ToSlash(resolved) + if created { + fmt.Fprint(cmd.OutOrStdout(), messages.CreatedCatalogFile(path)) + } + fmt.Fprint(cmd.OutOrStdout(), + messages.AddedToCatalog(kind, describeArtifact(ref), path)) + } + return nil +} + +// describeArtifact names what was recorded, with the published version when the +// job reported one, so a reader can pin it without going to look. +// +// Single-quoted to match the spec's transcripts; the surrounding Done: lines +// carry bare values, but a dataset name can hold a space and these cannot. +func describeArtifact(ref *project.ArtifactRef) string { + return messages.ArtifactDescription(ref.Name, ref.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go new file mode 100644 index 00000000000..9c9a91f4b8d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// catalogCommand builds a command whose output can be read back. +func catalogCommand(t *testing.T, buf *bytes.Buffer) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "generate"} + cmd.Flags().StringP("output", "o", "", "") + cmd.SetOut(buf) + return cmd +} + +// The spec's Scenario 2 transcript names what was recorded and the version it +// was published at. "Added catalog entry" says neither, which leaves the reader +// to infer both from the line above it. +func TestCatalogLineNamesTheArtifactAndVersion(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "support-agent-regression", + Source: "datasets/support-agent-regression.jsonl", + Version: "1", + })) + + out := buf.String() + assert.Contains(t, out, `Added dataset 'support-agent-regression' (version 1) to`) + assert.Contains(t, out, "eval.yaml") +} + +// The evaluator half of the same transcript. +func TestCatalogLineForAnEvaluator(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addEvaluatorToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "support-agent-quality", + Source: "evaluators/support-agent-quality.json", + Version: "1", + })) + + assert.Contains(t, buf.String(), `Added evaluator 'support-agent-quality' (version 1) to`) +} + +// A job that reported no version still has to name what it recorded, rather +// than printing an empty parenthesis or the word "latest" as if it were one. +func TestCatalogLineWithoutAVersion(t *testing.T) { + for _, version := range []string{"", "latest"} { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "golden", Source: "datasets/golden.jsonl", Version: version, + })) + + out := buf.String() + assert.Contains(t, out, `Added dataset 'golden' to`) + assert.NotContains(t, out, "version", "version %q is not one to print", version) + } +} + +// The first generate in a repository has no configuration to append to, so it +// says the file was created as well as what went into it. +func TestCatalogLineWhenTheFileIsCreated(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "golden", Source: "datasets/golden.jsonl", Version: "1", + })) + + out := buf.String() + assert.Contains(t, out, "Created") + assert.Contains(t, out, `Added dataset 'golden' (version 1) to`, + "creating the file still has to say what was put in it") + + // The entry is really on disk, not just announced. + body, err := os.ReadFile(filepath.Join(dir, project.EvalConfigBase)) + require.NoError(t, err) + assert.Contains(t, string(body), "golden") +} + +// Regenerating the same artifact to the same path changes nothing, so it must +// not claim it did. +func TestCatalogSaysNothingWhenNothingChanged(t *testing.T) { + dir := t.TempDir() + ref := &project.ArtifactRef{Name: "golden", Source: "datasets/golden.jsonl", Version: "1"} + + var first bytes.Buffer + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &first), dir, ref)) + require.Contains(t, first.String(), "Added dataset") + + var second bytes.Buffer + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &second), dir, ref)) + assert.Empty(t, second.String(), "an unchanged catalog is not an edit") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go new file mode 100644 index 00000000000..b6256a30565 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -0,0 +1,523 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "strings" + + "azureaieval/internal/foundry/projectctx" + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectEndpointEnvKey is the azd environment key holding the Foundry project +// endpoint the data-plane clients target. +const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + +// evalContext carries everything the commands need to reach the data plane. +type evalContext struct { + azdClient *azdext.AzdClient + endpoint string + envName string + cred azcore.TokenCredential + + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient + + // Held only once both listings succeed; a partial read is not reusable. + schemas map[string]*eval_api.EvaluatorSummary + + // Resolved on first use. Which command deploys cannot change while one + // command runs, and asking azd costs a round trip. + deployCmd string +} + +// newEvalContext resolves the project endpoint and builds the data-plane +// clients. The resolution order is projectctx's, so that every Foundry +// extension answers the same question the same way: +// +// 1. --project-endpoint +// 2. the active azd environment (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. global config: extensions.ai-agents.project.context.endpoint +// 4. the host environment variables of the same two names +// 5. otherwise an error naming how to set one +func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, messages.ConnectingToAzd(err) + } + + ec := &evalContext{azdClient: azdClient} + + // The environment name is resolved regardless of where the endpoint comes + // from: it is what the cached eval and run ids are read from and + // written to. Deriving it only when the endpoint came from azd meant + // --project-endpoint silently disabled that cache. + _, envName := lookupEndpointFromAzd(ctx, azdClient) + ec.envName = envName + + resolved, err := projectctx.Resolve(ctx, projectctx.ResolveOpts{FlagValue: endpointFlag}) + if err != nil { + // The caller only defers Close on a context it was handed, so every + // path that abandons this one has to close it here. + ec.Close() + return nil, err + } + ec.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") + log.Printf("[endpoint] resolved from %s", resolved.Source) + + cred, err := newAzdTokenCredential() + if err != nil { + ec.Close() + return nil, err + } + ec.cred = cred + + ec.evalClient = eval_api.NewEvalClient(ec.endpoint, ec.cred) + ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, ec.cred) + + return ec, nil +} + +// newAzdTokenCredential returns the azd credential already wrapped in its +// retry. Handing back the wrapper rather than the raw credential is what keeps +// the retry wired: an earlier version assigned the wrapper to the context and +// then built both clients from the unwrapped one, so nothing retried. +func newAzdTokenCredential() (azcore.TokenCredential, error) { + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, messages.CreatingCredential(err) + } + return azdTokenRetry{inner: cred}, nil +} + +// azdTokenRetry retries a failed token request once. azidentity gives the azd +// subprocess a fixed 10 second timeout and discards its stderr, so an azd that +// overruns surfaces as "exit status 1" with no cause; the next call usually +// finds a warm token. Without this a slow token turns into a failed command. +type azdTokenRetry struct{ inner azcore.TokenCredential } + +func (c azdTokenRetry) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + tok, err := c.inner.GetToken(ctx, opts) + if err == nil || ctx.Err() != nil { + return tok, err + } + log.Printf("[auth] token request failed (%v); retrying once", err) + return c.inner.GetToken(ctx, opts) +} + +// lookupEndpointFromAzd reads the endpoint from the active azd environment, +// returning empty strings when azd has no current environment. +// azdEnvironmentName is the environment this invocation acts on: the one +// -e/--environment named, or azd's current one when it named none. +// +// Answered here because it was answered independently in five places and +// -e was honoured by none of them. `azd ai eval create -e staging` read its +// endpoint out of the default environment and wrote its eval id back there, +// and `-e a-name-azd-rejects` was accepted in silence. +// +// Empty means there is no environment to act on, which is ordinary: the atomic +// commands work standalone against the data plane. +func azdEnvironmentName(ctx context.Context, azdClient *azdext.AzdClient) string { + if name := projectctx.SelectedEnvironment(ctx); name != "" { + return name + } + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp.GetEnvironment() == nil { + return "" + } + return envResp.Environment.Name +} + +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envName = azdEnvironmentName(ctx, azdClient) + if envName == "" { + return "", "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envName, + Key: projectEndpointEnvKey, + }) + if err != nil || val == nil || val.Value == "" { + return "", envName + } + return val.Value, envName +} + +// errNoAzdEnvironment reports that there is no azd environment to persist into. +// +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary rather than a problem worth reporting. +// A write that fails for any other reason still is. +var errNoAzdEnvironment = messages.ErrNoAzdEnvironment + +// remember persists a value that the extension can recover without, so a +// failure to store it must not fail the work that produced it. +// +// Running outside a project is ordinary -- the atomic commands are meant to +// work standalone against the data plane -- so having nowhere to write is not +// worth a word. Anything else is: these keys are how a later deploy recognizes +// what it already published, and losing one silently means the next `azd up` +// creates a second immutable version of something it had already created. +// +// Written to stderr, not through log: the standard logger is pointed at +// io.Discard unless --debug, so logging this would be the same silence with a +// more reassuring name. stderr keeps `-o json` on stdout parseable. azd does +// not surface an extension's stderr, so under `azd up` this reaches the debug +// log and no further -- direct invocations are where it shows. +func (ec *evalContext) remember(ctx context.Context, key, value string) { + err := ec.setEnvValue(ctx, key, value) + if err == nil || errors.Is(err, errNoAzdEnvironment) { + return + } + fmt.Fprint(os.Stderr, messages.Warning(err)) + log.Printf("[env] could not record %s: %v", key, err) +} + +// setEnvValue persists a value into the active azd environment. azd itself +// writes none of these keys -- the extension owns them. +func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { + if ec.envName == "" { + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return messages.NoAzdEnvironmentToWrite(key) + } + ec.envName = envResp.Environment.Name + } + _, err := ec.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: ec.envName, + Key: key, + Value: value, + }) + if err != nil { + return messages.WritingEnvValue(key, err) + } + return nil +} + +// confirmedNoAzdEnvironment reports that azd answered, and the answer was that +// there is no current environment. +// +// ec.envName being empty is not that answer. It is left empty by any failure to +// reach azd as well as by there being no environment, so reading it as "there +// is none" turns a transient gRPC hiccup into advice to create an environment +// the user already has. +// +// The environment name is recovered here when there turns out to be one, so a +// caller whose earlier lookup came up empty because of a hiccup can retry it. +func (ec *evalContext) confirmedNoAzdEnvironment(ctx context.Context) bool { + if ec.envName != "" { + return false + } + if ec.azdClient == nil { + // No azd to ask: running standalone against the data plane, where + // there is genuinely nowhere to have recorded an id. + return true + } + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil { + return isNoDefaultEnvironmentError(err) + } + if envResp == nil || envResp.Environment == nil || envResp.Environment.Name == "" { + return true + } + ec.envName = envResp.Environment.Name + return false +} + +// isNoDefaultEnvironmentError picks azd's "there is no environment to record +// anything in" out of every other reason the call could have failed. +// +// The distinction is the whole point: a transport failure must not be reported +// as a missing environment, or a gRPC hiccup tells the user to create one they +// already have. +// +// Written as the cascade's own rule minus the one case the two disagree on, +// rather than as a second list of azd's sentinels. Keeping a second list is how +// `no project exists` came to be handled in the cascade and missed here, which +// told anyone running outside a project to publish an eval that already exists. +func isNoDefaultEnvironmentError(err error) bool { + if err == nil { + return false + } + return projectctx.HostedSourceAbsent(err) && !projectctx.DaemonUnreachable(err) +} + +// getEnvValue reads a value from the active azd environment, returning empty +// when it is unset. +func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { + if ec.envName == "" || ec.azdClient == nil { + return "" + } + val, err := ec.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: ec.envName, + Key: key, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} + +// deployCommand names the command that publishes this project's evals. +// +// `azd up` provisions before it deploys, so it only works where there is +// infrastructure to provision. Evals are data-plane only, so a project that +// ships none fails compiling a missing infra/main.bicep and never reaches +// them -- naming `azd up` there hands the reader a failure instead of a fix. +func (ec *evalContext) deployCommand(ctx context.Context) string { + if ec.deployCmd != "" { + return ec.deployCmd + } + + proj, err := ec.azdProject(ctx) + name := deployCommandName(proj) + if err != nil { + // A project we could not read is not a project without infrastructure. + // Answer for this call, but do not cache what a transport failure said: + // one hiccup would otherwise downgrade the advice for the whole process. + return name + } + ec.deployCmd = name + return name +} + +// azdProject reads the project azd is running against. A nil project with no +// error means azd answered and there is none; an error means it did not answer. +func (ec *evalContext) azdProject(ctx context.Context) (*azdext.ProjectConfig, error) { + if ec.azdClient == nil { + return nil, nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil { + return nil, err + } + return resp.GetProject(), nil +} + +// deployCommandName is projectCanProvision phrased as the command to run. +// +// Without infrastructure the answer is this extension's own command rather than +// `azd deploy`. Deploy refuses with "infrastructure has not been provisioned" +// in an environment that has never provisioned one, which is exactly the +// scratch project `azd init --minimal` produces and an eval gets scaffolded +// into. `azd ai eval create` reconciles the same configuration needing nothing +// but an endpoint. +func deployCommandName(proj *azdext.ProjectConfig) string { + if projectCanProvision(proj) { + return azdUpCommand + } + return "azd ai eval create" +} + +// azdUpCommand provisions before it deploys. It is named rather than repeated +// because callers have to be able to tell it apart from this extension's own +// commands -- it takes none of their flags. +const azdUpCommand = "azd up" + +// appInsightsEnvKey is where a connected Application Insights resource lands in +// the azd environment. azd's own provisioning writes it, and the agents +// extension reads the same key to pass tracing configuration to a running +// agent, so its presence is the project's answer to "are traces being +// collected?". +const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +// defaultGenerationSource picks what `dataset generate` sends when --from was +// not given, from the Application Insights connection string the project has +// (or has not) been given. +// +// Traces are the better dataset when they exist, being real conversations +// rather than synthesized ones, so they win whenever the project is wired to +// collect them. Outside a project, or in one with no Application Insights, +// there are no traces to ask for and the agent's own definition is all that is +// left. +func defaultGenerationSource(appInsightsConnection string) []string { + if appInsightsConnection != "" { + return []string{project.GenerateFromTraces} + } + return []string{project.GenerateFromAgent} +} + +func (ec *evalContext) Close() { + if ec.azdClient != nil { + ec.azdClient.Close() + } +} + +// projectARMIDEnvKey holds the project's ARM resource ID, which is what the +// Foundry portal addresses a project by. azd provisioning writes it, and the +// agents extension reads the same key to build the same links. +const projectARMIDEnvKey = "AZURE_AI_PROJECT_ID" + +// portalPrefix builds the Foundry portal prefix for this project, or nil when +// the project cannot be addressed. +// +// Best effort by design: a portal link is a convenience on top of a command +// that already did its work, so a missing or unparseable resource ID drops the +// line rather than failing the command that earned it. +func (ec *evalContext) portalPrefix(ctx context.Context) *eval_api.PortalPrefix { + armID := ec.getEnvValue(ctx, projectARMIDEnvKey) + if armID == "" { + return nil + } + prefix, err := eval_api.NewPortalPrefix(armID) + if err != nil { + log.Printf("[portal] %s is not a project resource ID: %v", projectARMIDEnvKey, err) + return nil + } + return prefix +} + +// withPortalLink stamps a run with its portal URL, so the terminal and `-o json` +// answer with the same link from one place. +func (ec *evalContext) withPortalLink( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) *eval_api.OpenAIEvalRun { + if run == nil || evalID == "" || run.ID == "" { + return run + } + if prefix := ec.portalPrefix(ctx); prefix != nil { + run.PortalURL = prefix.EvalRunURL(evalID, run.ID) + } + return run +} + +// azd environment keys written by this extension. +const ( + envKeyEvalID = "EVAL_ID" + envKeyEvalRunID = "EVAL_RUN_ID" + envKeyDatasetVersion = "EVAL_DATASET_VERSION" + envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" + // envKeyEvalPath records where `init` put the configuration, so the + // commands that read it afterwards do not each need --path repeated. + envKeyEvalPath = "EVAL_CONFIG_PATH" +) + +// evalDirCascade is the one rule for where the configuration lives: +// +// 1. --path +// 2. the path `init` recorded in the azd environment +// 3. ./evals +// +// The middle level is what stops `--path` from having to be repeated on every +// later command. Without it, `init --path ./quality` wrote a configuration that +// `run` then looked for under ./evals and reported as missing -- while +// azure.yaml's $ref pointed at it correctly the whole time. +// +// This is the whole rule, and every command that reads the configuration goes +// through it. Stating it here and applying it on only some paths is how +// `create` came to report the configuration missing and `generate` came to +// write a second one under ./evals, both in a project where init had recorded +// where it put the first. +// +// recorded tells absence apart from failure, and the two get different +// answers. A project with no azd environment has genuinely recorded nothing, +// so ./evals is right. An azd that could not be asked has said nothing at all, +// and defaulting on that would write the second configuration all over again -- +// this time for a reason nobody could reproduce. +func evalDirCascade(flagValue string, recorded func() (string, error)) (string, error) { + if flagValue != "" { + return flagValue, nil + } + path, err := recorded() + if err != nil { + return "", err + } + if path != "" { + return path, nil + } + return project.DefaultEvalDir, nil +} + +// evalDir is the cascade for a command that already holds an azd connection. +func (ec *evalContext) evalDir(ctx context.Context, flagValue string) (string, error) { + return evalDirCascade(flagValue, func() (string, error) { + if ec.envName == "" || ec.azdClient == nil { + return "", nil + } + return readRecordedEvalPath(ctx, ec.azdClient, ec.envName) + }) +} + +// resolveEvalDir is the cascade for a command that has not built an +// evalContext yet. +// +// `create`, `generate` and `init` all have to find the configuration before +// they resolve a Foundry endpoint, so that a project with no configuration is +// told to run `init` rather than told to set an endpoint. The extra azd +// connection is local and short-lived, and is what buys that ordering. +func resolveEvalDir(ctx context.Context, flagValue string) (string, error) { + return evalDirCascade(flagValue, func() (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + // Nothing to ask. This extension is spawned by azd, so the case + // that reaches here is a test or a direct invocation, neither of + // which has an environment holding a recorded path. + return "", nil + } + defer azdClient.Close() + + if name := projectctx.SelectedEnvironment(ctx); name != "" { + return readRecordedEvalPath(ctx, azdClient, name) + } + + env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil { + // The same rule the endpoint cascade uses: azd saying "there is no + // project or environment" is an answer, and anything else is not. + if isNoDefaultEnvironmentError(err) { + return "", nil + } + return "", err + } + if env.GetEnvironment() == nil { + return "", nil + } + return readRecordedEvalPath(ctx, azdClient, env.GetEnvironment().GetName()) + }) +} + +// readRecordedEvalPath reads back what recordEvalPath wrote, distinguishing a +// key that was never set from a read that failed. +func readRecordedEvalPath( + ctx context.Context, + client *azdext.AzdClient, + envName string, +) (string, error) { + val, err := client.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envName, + Key: envKeyEvalPath, + }) + if err != nil { + // An unset key is an answer; init records the path best effort and + // succeeds without an azd environment to record it in. + if isNoDefaultEnvironmentError(err) { + return "", nil + } + return "", err + } + if val == nil { + return "", nil + } + return val.Value, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go new file mode 100644 index 00000000000..49b57cd005a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingCred fails its first failUntil calls, then succeeds. +type countingCred struct { + calls int + failFor int + lastOpts policy.TokenRequestOptions +} + +func (c *countingCred) GetToken( + _ context.Context, opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + c.calls++ + c.lastOpts = opts + if c.calls <= c.failFor { + return azcore.AccessToken{}, errors.New("AzureDeveloperCLICredential: exit status 1") + } + return azcore.AccessToken{Token: "token"}, nil +} + +// The tests below construct azdTokenRetry directly, so they all pass even if +// nothing wires it in. This one guards the wiring: an earlier version assigned +// the wrapper to the context and built both clients from the raw credential, +// which made the retry dead code. +func TestNewAzdTokenCredentialReturnsTheRetryingCredential(t *testing.T) { + cred, err := newAzdTokenCredential() + require.NoError(t, err) + _, wrapped := cred.(azdTokenRetry) + assert.True(t, wrapped, "clients must be built from the retrying credential, not the raw one") +} + +// The failure this retries carries no cause: azidentity kills the azd +// subprocess at a fixed 10s and discards its stderr. Retrying is the only way +// to tell a slow token from a broken login. +func TestAzdTokenRetryRecoversFromOneFailure(t *testing.T) { + inner := &countingCred{failFor: 1} + + tok, err := azdTokenRetry{inner: inner}.GetToken( + t.Context(), policy.TokenRequestOptions{Scopes: []string{"scope"}}) + + require.NoError(t, err, "a token that succeeds on the second attempt must not fail the command") + assert.Equal(t, "token", tok.Token) + assert.Equal(t, 2, inner.calls, "exactly one retry") + assert.Equal(t, []string{"scope"}, inner.lastOpts.Scopes, "the retry keeps the caller's scopes") +} + +func TestAzdTokenRetryDoesNotRetryASuccess(t *testing.T) { + inner := &countingCred{} + _, err := azdTokenRetry{inner: inner}.GetToken(t.Context(), policy.TokenRequestOptions{}) + require.NoError(t, err) + assert.Equal(t, 1, inner.calls, "a working token costs one call") +} + +// Retrying must not paper over a genuine failure, and must stop at one. +func TestAzdTokenRetryGivesUpAfterTheSecondFailure(t *testing.T) { + inner := &countingCred{failFor: 99} + _, err := azdTokenRetry{inner: inner}.GetToken(t.Context(), policy.TokenRequestOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "exit status 1", "the original cause survives") + assert.Equal(t, 2, inner.calls, "no more than one retry") +} + +// A cancelled context is the user pressing Ctrl+C or a deadline expiring; +// retrying there would just fail again more slowly. +func TestAzdTokenRetryDoesNotRetryACancelledContext(t *testing.T) { + inner := &countingCred{failFor: 99} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := azdTokenRetry{inner: inner}.GetToken(ctx, policy.TokenRequestOptions{}) + require.Error(t, err) + assert.Equal(t, 1, inner.calls, "a cancelled context is not retried") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go new file mode 100644 index 00000000000..055bb60e1ac --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -0,0 +1,439 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// firstDatasetVersions are the versions a dataset's first publish can carry, +// probed when the version listing has not caught up yet. +// +// The service assigns nothing; the client picks. This CLI's first publish is +// NextVersion(""), so probing a hardcoded "1" never found a dataset this CLI +// had just created -- which is the one case the probe exists for. "1" is still +// probed because a generation job, the SDK or the portal can register one. +var firstDatasetVersions = []string{dataset_api.NextVersion(""), "1"} + +func newDatasetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage evaluation datasets.", + } + cmd.AddCommand( + newDatasetCreateCommand(), + newDatasetUpdateCommand(), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + newDatasetVersionsCommand(), + ) + return cmd +} + +// newDatasetCreateCommand builds `dataset create `, which registers a +// dataset that does not exist yet. +func newDatasetCreateCommand() *cobra.Command { + return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") +} + +// newDatasetUpdateCommand builds `dataset update `, which publishes a +// further version of one that does. +func newDatasetUpdateCommand() *cobra.Command { + return newDatasetWriteCommand("update", "Publish a new version of a dataset.") +} + +// datasetPresence answers whether the dataset is already registered, and +// whether a "no" can be trusted. +// +// The version listing lags a publish, so a create followed straight by an +// update was told the dataset it had just made does not exist. A point read of +// the versions a first publish can carry usually settles that, catching up +// sooner than the listing. +// +// Absence is only certain when the listing itself answered 404. An empty 200 +// does not prove it: an unknown dataset and a listing that has not caught up +// are indistinguishable. +// +// A read that failed for any other reason proves nothing at all, and is +// returned. Treating a 403 or a timeout as "not there" let `create` go on to +// publish a further version of a dataset that already existed -- the one thing +// separating create from update, decided by an error nobody looked at. +func datasetPresence( + ctx context.Context, + client *dataset_api.DatasetClient, + name string, +) (exists, absenceCertain bool, err error) { + existing, listErr := client.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if listErr != nil && !dataset_api.IsNotFound(listErr) { + return false, false, messages.CheckingDataset(name, listErr) + } + if listErr == nil && existing != nil && len(existing.Value) > 0 { + return true, false, nil + } + + for _, v := range firstDatasetVersions { + _, getErr := client.GetDataset(ctx, name, v, ProjectEndpointAPIVersion) + if getErr == nil { + return true, false, nil + } + if !dataset_api.IsNotFound(getErr) { + return false, false, messages.CheckingDataset(name, getErr) + } + } + return false, dataset_api.IsNotFound(listErr), nil +} + +// newDatasetWriteCommand builds create and update. Both run the same upload, +// and the existence check is the only thing that separates them: a version is +// brought into being by startPendingUpload, which neither knows nor cares +// whether the name was already in use. +func newDatasetWriteCommand(verb, short string) *cobra.Command { + var ( + fromFile string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: verb + " ", + Short: short, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } + if fromFile == "" { + return requireFlag("from-file") + } + + localSource, err := datasetUploadSource(fromFile) + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + exists, absenceCertain, err := datasetPresence(ctx, ec.datasetClient, name) + if err != nil { + return err + } + if err := checkAssetExistence( + verb, "dataset", name, exists, absenceCertain, + ); err != nil { + return err + } + + ds, err := ec.datasetClient.UploadNextVersion( + ctx, name, version, localSource, ProjectEndpointAPIVersion, + ) + if err != nil { + return messages.RegisteringDataset(name, err) + } + + if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { + // Persisting is a convenience, so this never fails the command. + // It goes to stdout because azd does not surface an extension's + // stderr, and is skipped outside a project, where having nowhere + // to persist is expected rather than notable. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprint(cmd.OutOrStdout(), messages.Warning(err)) + } + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + fmt.Fprint(cmd.OutOrStdout(), messages.DatasetRegistered(ds.Name, ds.Version)) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Path to a .jsonl file, or a directory containing one.") + // Only on update. create publishes a first version, and the upload derives + // the next version from whatever this holds, so `create --version 4.0` + // would publish 5.0 -- and leave the existence probe, which looks for the + // versions a first publish can carry, unable to find what it wrote. + if verb == "update" { + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to increment from the latest registered version.") + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// datasetUploadSource resolves what was named into the path the upload reads. +// +// A named file is returned as itself. Returning its directory would upload +// whichever .jsonl sorts first, so pointing --from-file at one dataset in a +// folder holding several would register a different one under that name — and +// the fingerprint would describe the file that was named, so the two would +// agree forever afterwards. +// +// A directory is resolved to the single .jsonl inside it, which is what the +// flag offers. Several is not that, and picking one would be a guess. +func datasetUploadSource(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", messages.ReadingFromFile(path, err) + } + if !info.IsDir() { + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", messages.FromFileMustBeJSONL(path) + } + return path, nil + } + + entries, err := os.ReadDir(path) + if err != nil { + return "", messages.ReadingFromFile(path, err) + } + var found []string + for _, e := range entries { + if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".jsonl") { + found = append(found, e.Name()) + } + } + switch len(found) { + case 0: + return "", messages.FromFileDirectoryHasNoJSONL(path) + case 1: + return filepath.Join(path, found[0]), nil + default: + return "", messages.FromFileDirectoryIsAmbiguous(path, found) + } +} + +func newDatasetListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's datasets.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) + if err != nil { + return messages.ListingDatasets(err) + } + return renderDatasets(cmd, list, messages.NoDatasets()) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newDatasetVersionsCommand groups the version listing, so that `list` means +// the assets rather than the history of one of them. +func newDatasetVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one dataset.", + } + cmd.AddCommand(newDatasetVersionsListCommand()) + return cmd +} + +func newDatasetVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of a dataset.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return messages.ListingDatasetVersions(name, err) + } + // An unknown name lists nothing and succeeds; it is not an error. + // `-o json` callers range over the array, and `dataset delete` is + // checked for idempotence by listing what is left. The empty sentence + // names the dataset, though: the project may hold plenty of others, so + // "No datasets found." would be answering a different question. + return renderDatasets(cmd, list, messages.NoDatasetVersions(name)) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList, whenEmpty string) error { + // JSON is decided before emptiness: a caller piping this into a parser needs + // an empty array, not the sentence a human would read. + if list == nil { + list = &dataset_api.DatasetList{} + } + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Type}) + } + if len(rows) == 0 { + fmt.Fprint(cmd.OutOrStdout(), whenEmpty) + return nil + } + // TYPE, not FORMAT: the service populates type (`uri_file`) and leaves + // format empty, so the column was blank on every row. + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) +} + +func newDatasetShowCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if version == "" { + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return messages.ResolvingLatestDatasetVersion(name, err) + } + if len(list.Value) == 0 { + // The service answers an unknown name with an empty list + // rather than a 404, and a dataset cannot exist with no + // versions, so this is what "no such dataset" looks like. + return messages.DatasetNotFound(name) + } + version = dataset_api.LatestVersion(list.Value) + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + if eval_api.IsNotFound(err) { + return messages.DatasetVersionNotFoundWithHint(name, version) + } + return messages.ReadingDatasetVersion(name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + if err := emitDetail(cmd.OutOrStdout(), []field{ + {"Name", ds.Name}, + {"Version", ds.Version}, + {"Type", ds.Type}, + {"URI", ds.ResolvedBlobURI()}, + }); err != nil { + return err + } + if prefix := ec.portalPrefix(ctx); prefix != nil { + writePortalLink(cmd.OutOrStdout(), prefix.DatasetURL(ds.Name, ds.Version)) + } + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetDeleteCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.datasetClient.DeleteDatasetVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + if eval_api.IsNotFound(err) { + return messages.DatasetVersionNotFound(name, version) + } + return messages.DeletingDatasetVersion(name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.DatasetDeleted(name, version)) + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go new file mode 100644 index 00000000000..388c8d21106 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An empty listing says which listing was empty. +// +// The caller already passed the sentence to print, and this printed a different +// one: `dataset versions list ` answered "No datasets found", a report +// about the whole project, for a name that simply had no versions. The +// parameter was threaded through and then ignored, so the fix that added it +// changed nothing and no test noticed. +func TestEmptyDatasetListingUsesTheCallersMessage(t *testing.T) { + cases := []struct { + name string + whenEmpty string + }{ + {"whole project", messages.NoDatasets()}, + {"one dataset's versions", messages.NoDatasetVersions("golden")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&out) + + require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{}, tc.whenEmpty)) + assert.Equal(t, tc.whenEmpty, out.String()) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go new file mode 100644 index 00000000000..4ba2e04e7fb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// presenceServer stands up a fake project endpoint and records every path the +// presence probe asks for, so a test can assert what was tried as well as what +// was concluded. +type presenceServer struct { + mu sync.Mutex + paths []string +} + +// requested returns the paths seen so far, in order. +func (s *presenceServer) requested() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.paths...) +} + +// newPresenceClient wires a DatasetClient to a server that answers the version +// listing with listStatus/listBody, and answers a point read of a dataset +// version with whatever found reports for that version. +// +// Retries are off: a test that means "the service said 404" should cost one +// request, not a backoff schedule. +func newPresenceClient( + t *testing.T, + listStatus int, + listBody string, + found map[string]bool, +) (*dataset_api.DatasetClient, *presenceServer) { + t.Helper() + + rec := &presenceServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.mu.Lock() + rec.paths = append(rec.paths, r.URL.Path) + rec.mu.Unlock() + + // Assertions inside a handler run on the server's goroutine, where a + // Fatalf would leave the client hanging on a response never written. + assert.Equal(t, http.MethodGet, r.Method) + + switch { + case strings.HasSuffix(r.URL.Path, "/versions"): + w.WriteHeader(listStatus) + _, _ = w.Write([]byte(listBody)) + default: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if found[version] { + w.WriteHeader(http.StatusOK) + // A fixed body: datasetPresence reads only whether the point + // read succeeded, and echoing the request path back would make + // this a taint sink for no benefit. + _, _ = w.Write([]byte(`{"name":"ds"}`)) + return + } + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + client := dataset_api.NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + return client, rec +} + +// TestPresenceTrustsANonEmptyVersionListing is the ordinary case: the listing +// answered, so no point read is needed. +func TestPresenceTrustsANonEmptyVersionListing(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[{"name":"ds","version":"1.0"}]}`, nil) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) + + require.True(t, exists) + require.False(t, absenceCertain) + require.Equal(t, []string{"/datasets/ds/versions"}, rec.requested(), + "a listing that answered should settle it without a point read") +} + +// TestPresenceProbesPastAListingThatHasNotCaughtUp covers the bug the probe +// exists for: a create publishes 1.0, the listing still reports nothing, and +// the update that follows must not be told the dataset is missing. +func TestPresenceProbesPastAListingThatHasNotCaughtUp(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[]}`, map[string]bool{"1.0": true}) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) + + require.True(t, exists, "the point read found the version the listing had not") + require.False(t, absenceCertain) + require.Equal(t, []string{"/datasets/ds/versions", "/datasets/ds/versions/1.0"}, + rec.requested()) + require.NoError(t, checkAssetExistence("update", "dataset", "ds", exists, absenceCertain)) + require.Error(t, checkAssetExistence("create", "dataset", "ds", exists, absenceCertain), + "create must still refuse a name the probe found") +} + +// TestPresenceProbesTheVersionSomethingElseRegistered covers a dataset created +// by the portal, the SDK or a generation job, which numbers its first version +// "1" rather than the "1.0" this CLI publishes. +func TestPresenceProbesTheVersionSomethingElseRegistered(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[]}`, map[string]bool{"1": true}) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) + + require.True(t, exists) + require.False(t, absenceCertain) + require.Equal(t, + []string{"/datasets/ds/versions", "/datasets/ds/versions/1.0", "/datasets/ds/versions/1"}, + rec.requested(), + "both first-publish versions should be probed before giving up") +} + +// TestPresenceWillNotCallAnEmptyListingProofOfAbsence is the guard that keeps +// `update` working against a service whose listing lags. An empty 200 is not a +// 404, so the gate must let the update through rather than refuse it. +func TestPresenceWillNotCallAnEmptyListingProofOfAbsence(t *testing.T) { + client, _ := newPresenceClient(t, http.StatusOK, `{"value":[]}`, nil) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) + + require.False(t, exists) + require.False(t, absenceCertain, + "an empty listing does not distinguish an unknown dataset from a stale one") + require.NoError(t, checkAssetExistence("update", "dataset", "ds", exists, absenceCertain), + "update must proceed when absence is unproven") +} + +// TestPresenceTreatsA404ListingAsProofOfAbsence is the other half: a service +// that actually said "no such dataset" should stop an update before it uploads. +func TestPresenceTreatsA404ListingAsProofOfAbsence(t *testing.T) { + client, _ := newPresenceClient(t, + http.StatusNotFound, `{"error":{"code":"NotFound"}}`, nil) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) + + require.False(t, exists) + require.True(t, absenceCertain) + + gateErr := checkAssetExistence("update", "dataset", "ds", exists, absenceCertain) + require.Error(t, gateErr) + require.Contains(t, gateErr.Error(), `dataset "ds" does not exist`) + require.NoError(t, checkAssetExistence("create", "dataset", "ds", exists, absenceCertain), + "create is exactly what a proven-absent name should allow") +} + +// A read that failed proves nothing. Answering "not there" let `create` publish +// a further version of a dataset that already existed -- the one thing +// separating create from update, decided by an error nobody looked at. +func TestPresenceReportsAListingThatFailedRatherThanGuessing(t *testing.T) { + for _, status := range []int{ + http.StatusForbidden, + http.StatusUnauthorized, + http.StatusTooManyRequests, + http.StatusInternalServerError, + } { + t.Run(http.StatusText(status), func(t *testing.T) { + client, _ := newPresenceClient(t, status, `{"error":{"code":"Nope"}}`, nil) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + + require.Error(t, err, "a failed listing is not an answer about existence") + require.False(t, exists) + require.False(t, absenceCertain) + require.Contains(t, err.Error(), "ds") + }) + } +} + +// The same rule on the point read: only a 404 means "not this version". +func TestPresenceReportsAPointReadThatFailedRatherThanGuessing(t *testing.T) { + rec := &presenceServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.mu.Lock() + rec.paths = append(rec.paths, r.URL.Path) + rec.mu.Unlock() + + if strings.HasSuffix(r.URL.Path, "/versions") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"value":[]}`)) + return + } + http.Error(w, `{"error":{"code":"Forbidden"}}`, http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + client := dataset_api.NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + + require.Error(t, err) + require.False(t, exists) + require.False(t, absenceCertain) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go new file mode 100644 index 00000000000..dd25bcf022b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadJSONLBytes(t *testing.T) { + content := []byte( + "{\"query\":\"a\"}\n" + + "\n" + // blank lines are skipped, not treated as rows + "{\"query\":\"b\"}\n" + + " {\"query\":\"c\"} \n") + + all, err := readJSONLBytes(content, 0) + require.NoError(t, err) + require.Len(t, all, 3) + assert.Equal(t, "a", all[0]["query"]) + assert.Equal(t, "c", all[2]["query"], "surrounding whitespace is not part of the row") +} + +// The limit is what makes --max-samples mean the same thing for a published +// dataset as for a local file. +func TestReadJSONLBytes_StopsAtTheLimit(t *testing.T) { + content := []byte("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + + two, err := readJSONLBytes(content, 2) + require.NoError(t, err) + require.Len(t, two, 2) + assert.EqualValues(t, 1, two[0]["n"]) + assert.EqualValues(t, 2, two[1]["n"]) + + // A limit larger than the file is not an error. + more, err := readJSONLBytes(content, 99) + require.NoError(t, err) + assert.Len(t, more, 3) +} + +func TestReadJSONLBytes_ReportsTheOffendingLine(t *testing.T) { + _, err := readJSONLBytes([]byte("{\"n\":1}\nnot json\n"), 0) + require.ErrorContains(t, err, "line 2") +} + +func TestReadJSONLBytes_EmptyIsNotAnError(t *testing.T) { + items, err := readJSONLBytes([]byte("\n\n"), 0) + require.NoError(t, err) + assert.Empty(t, items, "the caller decides whether no rows is a problem") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go new file mode 100644 index 00000000000..bdf9ec25eec --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A named file has to be uploaded as itself. +// +// This resolver used to return the file's DIRECTORY, and the upload then took +// whichever .jsonl sorted first. Pointing --from-file at one dataset in a +// folder holding several therefore registered a different one under that name, +// and because the fingerprint described the file that was named, the two agreed +// with each other forever afterwards. The sibling dataset extension was fixed; +// this copy was not, so the same command uploaded different bytes depending on +// which namespace the user typed. +func TestDatasetUploadSourceReadsTheFileThatWasNamed(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "a-sorts-first.jsonl") + wanted := filepath.Join(dir, "b-is-the-one-named.jsonl") + require.NoError(t, os.WriteFile(first, []byte(`{"query":"a"}`), 0o600)) + require.NoError(t, os.WriteFile(wanted, []byte(`{"query":"b"}`), 0o600)) + + got, err := datasetUploadSource(wanted) + + require.NoError(t, err) + assert.Equal(t, wanted, got, "the file that was named, not the one that sorts first") +} + +// A directory is offered by the flag, so one .jsonl inside it resolves. +func TestDatasetUploadSourceResolvesADirectoryHoldingOne(t *testing.T) { + dir := t.TempDir() + only := filepath.Join(dir, "only.jsonl") + require.NoError(t, os.WriteFile(only, []byte(`{"query":"a"}`), 0o600)) + + got, err := datasetUploadSource(dir) + + require.NoError(t, err) + assert.Equal(t, only, got) +} + +// Several is not "a directory containing one", and picking would be a guess. +func TestDatasetUploadSourceRefusesAnAmbiguousDirectory(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.jsonl"), []byte(`{}`), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.jsonl"), []byte(`{}`), 0o600)) + + _, err := datasetUploadSource(dir) + + require.Error(t, err) + assert.Contains(t, err.Error(), "a.jsonl") + assert.Contains(t, err.Error(), "b.jsonl", "naming them is what makes the refusal actionable") +} + +func TestDatasetUploadSourceRefusesADirectoryWithNoJSONL(t *testing.T) { + _, err := datasetUploadSource(t.TempDir()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no .jsonl file") +} + +// The service refuses a bad name with a 400 wrapping four levels of JSON, so +// the guard exists to say it plainly. The sibling extension had it; this copy, +// which serves the same commands under `azd ai eval dataset`, did not. +func TestValidAssetNameMatchesTheSibling(t *testing.T) { + for _, ok := range []string{"golden", "a_b-c", "A1"} { + assert.Truef(t, validAssetName(ok), "%q is a name the service accepts", ok) + } + for _, bad := range []string{"", "has space", "slash/name", "dots.here", "uni\u00e9"} { + assert.Falsef(t, validAssetName(bad), "%q must be refused before the round trip", bad) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go new file mode 100644 index 00000000000..53519c47e8b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// A version is what an eval binds to, so publishing must always add one and +// never change one that exists. Evaluators needed a guard for that; this is +// the same question asked of datasets, against the real service. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// liveDatasetClient builds a dataset client against the live project. +func liveDatasetClient(t *testing.T) *dataset_api.DatasetClient { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := os.Getenv("FOUNDRY_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "FOUNDRY_PROJECT_ENDPOINT is required") + + cred, err := liveCredential() + require.NoError(t, err) + return dataset_api.NewDatasetClient(endpoint, retryingCredential{inner: cred}) +} + +// writeRows puts a one-row JSONL file in its own directory, which is what the +// upload path reads from. +func writeRows(t *testing.T, answer string) string { + t.Helper() + dir := t.TempDir() + row := fmt.Sprintf(`{"query":"q","response":%q}`+"\n", answer) + require.NoError(t, os.WriteFile(filepath.Join(dir, "rows.jsonl"), []byte(row), 0o600)) + return dir +} + +// TestLiveDatasetVersionIsNeverOverwritten publishes at a version that already +// exists and requires the service to refuse. +// +// The reconciler relies on exactly this: when an author pins `version:` and +// the local content has changed, it publishes at that version and treats a +// conflict as the signal to stop. If the service accepted the write instead, +// the pinned version would silently change under every eval bound to it, and +// `azd up` would report success. +func TestLiveDatasetVersionIsNeverOverwritten(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_immutable_%d", time.Now().UnixNano()) + + first, err := client.UploadVersion( + ctx, name, "1", writeRows(t, "original"), ProjectEndpointAPIVersion) + require.NoError(t, err) + require.Equal(t, "1", first.Version) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, "1", ProjectEndpointAPIVersion) + }) + + _, err = client.UploadVersion( + ctx, name, "1", writeRows(t, "replacement"), ProjectEndpointAPIVersion) + require.Error(t, err, + "publishing over an existing dataset version must be refused, not accepted") + assert.True(t, dataset_api.IsVersionConflict(err), + "the refusal must be a conflict the reconciler can recognize; got: %v", err) +} + +// TestLiveDatasetUpdateAddsAVersion is the other half: the ordinary path must +// keep adding versions rather than reusing the newest. +func TestLiveDatasetUpdateAddsAVersion(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_next_%d", time.Now().UnixNano()) + + first, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "one"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + + // Immediate, because the version listing lags a publish and this is the + // window where a second upload could be told the dataset is new and + // restart at the version the first one just took. + second, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "two"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + + assert.NotEqual(t, first.Version, second.Version, + "a second upload must add a version rather than reuse the first") + + // Both readable, and the first still holding what it was published with. + original, err := client.GetDataset(ctx, name, first.Version, ProjectEndpointAPIVersion) + require.NoError(t, err) + assert.NotEmpty(t, original.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go new file mode 100644 index 00000000000..fa238574328 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" +) + +// The existence probe exists for one case: `create` then immediately `update`, +// where the version listing has not caught up. It probed a hardcoded "1" while +// this CLI's first publish is NextVersion(""), which is "1.0" -- so for the +// case it was written for it read a version that never existed and the fallback +// was inert. Deriving it keeps the two in step if the base ever moves. +func TestFirstDatasetVersionsCoverWhatThisCLIPublishes(t *testing.T) { + assert.Contains(t, firstDatasetVersions, dataset_api.NextVersion(""), + "the probe has to look for the version a create actually writes") + assert.Contains(t, firstDatasetVersions, "1", + "a generation job, the SDK or the portal can register a plain 1") +} + +// The probe still cannot prove absence -- it only ever proves existence, and a +// dataset whose early versions were deleted has none of them left to find. So +// an absence the service never confirmed has to publish rather than refuse, +// otherwise the caller is sent to `create`, which fails in turn once the +// listing catches up and reports the name already taken. +func TestCheckAssetExistenceLetsAnUnprovableAbsenceThrough(t *testing.T) { + assert.NoError(t, checkAssetExistence("update", "dataset", "x", false, false)) + + err := checkAssetExistence("update", "dataset", "x", false, true) + assert.Error(t, err, "a 404 is the service saying the name is unknown, which still refuses") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go new file mode 100644 index 00000000000..6934bc01997 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "time" + + azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/spf13/pflag" +) + +// setupDebugLogging silences the standard logger unless debug mode is on. +// +// The data-plane clients trace every request through log.Printf, which Go +// writes to stderr by default. Without this the CLI interleaves raw HTTP traces +// with its own output on every command. +// +// The returned function puts the logger back and closes the file. Callers run +// for the length of the process and let the OS close it, so it is returned for +// tests and for any caller that wants to stop logging early. +func setupDebugLogging(flags *pflag.FlagSet) func() { + if !isDebug(flags) { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + return func() {} + } + + // Written outside the working directory: that is the user's repository, the + // scaffolded .gitignore does not cover this name, and a routine `git add -A` + // committed one. + // + // The name is picked by CreateTemp rather than built from the date alone. + // The temp directory is shared on Linux, and at a predictable path another + // user can leave a file of their own -- readable, to collect HTTP traces + // that carry request headers, or a symbolic link, to have them written to a + // file of their choosing. CreateTemp finds an unused name and creates it + // 0600 in one step, so neither is reachable. The date stays in the name + // because it is what makes a directory of these readable, and the full path + // is echoed below. + logFile, err := os.CreateTemp("", fmt.Sprintf("azd-ai-eval-%s-*.log", time.Now().Format("2006-01-02"))) + + var w io.Writer + var closeFile func() + if err != nil { + w = os.Stderr + closeFile = func() {} + } else { + w = logFile + closeFile = func() { _ = logFile.Close() } + // A log nobody can find is not a log. Debugging was asked for + // explicitly, so naming the file costs nothing. + fmt.Fprintf(os.Stderr, "Debug log: %s\n", filepath.ToSlash(logFile.Name())) + } + + log.SetOutput(w) + azcorelog.SetListener(func(event azcorelog.Event, msg string) { + fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) + }) + + return func() { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + closeFile() + } +} + +// isDebug reports whether --debug or AZD_EXT_DEBUG is set. +func isDebug(flags *pflag.FlagSet) bool { + if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { + return true + } + debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) + return debug +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go new file mode 100644 index 00000000000..5f60108f92d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// The create request has no description field, so a documented description +// would otherwise be parsed and dropped. +func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") + group.Description = "Quality gate for the support agent" + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) +} + +// An absent description adds no metadata key rather than an empty one. +func TestBuildOmitsEmptyDescription(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.NotContains(t, req.Metadata, "azd_description") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go new file mode 100644 index 00000000000..75986efd373 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's output conventions split the two shapes: a list view is uppercase +// headers over a rule, a detail view is Title Case key/value. `show` returns +// one thing, so it is a detail view -- and all three used to disagree, two +// emitting raw JSON whatever was asked for and one printing a one-row table. +// +// Checked by reading the source rather than running the command, because these +// commands need a service; a shape regression should not wait for a live run. +func TestShowCommandsUseDetailViews(t *testing.T) { + // Command → the file and function that renders it. + renderers := map[string]string{ + "dataset show": "dataset.go", + "evaluator show": "evaluator.go", + "show": "eval_group.go", + } + + for path, file := range renderers { + t.Run(path, func(t *testing.T) { + require.NotNil(t, find(t, path), "the command has to exist to have a shape") + + body, err := os.ReadFile(filepath.Join(".", file)) + require.NoError(t, err) + assert.Containsf(t, string(body), "emitDetail", + "%s returns one thing, so %s renders it as a detail view", path, file) + }) + } +} + +// Every command returning data supports -o json, which is what makes the +// detail view a presentation choice rather than a loss of information. +func TestShowCommandsStillAnswerInJSON(t *testing.T) { + for _, path := range []string{"dataset show", "evaluator show", "show"} { + cmd := find(t, path) + // -o comes from the SDK root, so a command must not shadow it. + assert.Nilf(t, cmd.LocalFlags().Lookup("output"), + "%s must inherit -o rather than declaring its own", path) + } +} + +// A detail view is two columns with Title Case keys, per the spec's output +// conventions. It is what `show` prints; `-o json` is the machine-readable +// alternative, not the only form. +func TestEmitDetail(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Name", "support-quality"}, + {"Version", "3"}, + {"Type", "rubric"}, + })) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3) + for i, want := range []string{"Name", "Version", "Type"} { + assert.Truef(t, strings.HasPrefix(lines[i], want), + "line %d should start with the key %q, got %q", i, want, lines[i]) + } + assert.Contains(t, lines[0], "support-quality") +} + +// A blank value says only that the writer did not know which fields this kind +// has, so it is dropped rather than printed as an empty column. +func TestEmitDetail_DropsEmptyValues(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Name", "support-quality"}, + {"Description", ""}, + {"Type", "rubric"}, + })) + + assert.NotContains(t, buf.String(), "Description") + assert.Len(t, strings.Split(strings.TrimRight(buf.String(), "\n"), "\n"), 2) +} + +// `evaluator show` used to emit raw JSON whatever was asked for, which left it +// the one `show` with no detail view and nowhere to put a portal link. +func TestRenderEvaluator(t *testing.T) { + var buf bytes.Buffer + ec := &evalContext{} + + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, &eval_api.EvaluatorSummary{ + Name: "support-quality", + Version: "3", + EvaluatorType: "rubric", + Description: "Grades politeness and accuracy.", + Categories: []string{"quality", "custom"}, + SupportedEvaluationLevels: []string{"turn", "conversation"}, + })) + + out := buf.String() + for _, want := range []string{ + "Name", "support-quality", + "Version", "3", + "Type", "rubric", + "Description", "Grades politeness and accuracy.", + "Categories", "quality, custom", + "Evaluation Levels", "turn, conversation", + } { + assert.Contains(t, out, want) + } + + // The schemas live in -o json: printed here they would bury the few lines a + // reader came for. + assert.NotContains(t, out, "data_schema") + assert.NotContains(t, out, "init_parameters") +} + +// Both spellings of the type field are read, because the listing says +// evaluator_type and other payloads say type. +func TestRenderEvaluator_ReadsEitherTypeSpelling(t *testing.T) { + for _, e := range []*eval_api.EvaluatorSummary{ + {Name: "x", EvaluatorType: "rubric"}, + {Name: "x", TypeAlias: "rubric"}, + } { + var buf bytes.Buffer + ec := &evalContext{} + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, e)) + assert.Contains(t, buf.String(), "rubric") + } +} + +// Without an azd environment there is no project to address, so the view ends +// at its last field rather than at an empty label. +func TestRenderEvaluator_NoPortalLinkWithoutAProject(t *testing.T) { + var buf bytes.Buffer + ec := &evalContext{} + + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, + &eval_api.EvaluatorSummary{Name: "support-quality", Version: "3"})) + + assert.NotContains(t, buf.String(), "Portal:") +} + +// The spec gives evaluator URLs their own shape, distinct from datasets and +// runs, so a link built from the wrong one resolves to nothing. +func TestPortalEvaluatorURLShape(t *testing.T) { + prefix, err := eval_api.NewPortalPrefix( + "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + assert.True(t, strings.HasSuffix( + prefix.EvaluatorURL("support-quality", "3"), + "/build/evaluations/catalog/support-quality/3")) +} + +// The pass mark is what Scenario 4 changes between versions, so a version +// listing that omits it cannot answer the question it is read for. +func TestEvaluatorPassThreshold(t *testing.T) { + threshold := func(v float64) *eval_api.EvaluatorContract { + return &eval_api.EvaluatorContract{PassThreshold: &v} + } + + cases := []struct { + name string + in *eval_api.EvaluatorSummary + want string + }{ + {"absent evaluator", nil, ""}, + {"no definition", &eval_api.EvaluatorSummary{}, ""}, + { + "definition without a threshold", + &eval_api.EvaluatorSummary{Definition: &eval_api.EvaluatorContract{}}, + "", + }, + { + "a threshold of zero is a real threshold, not an absent one", + &eval_api.EvaluatorSummary{Definition: threshold(0)}, + "0.00", + }, + { + "two decimals, because 0.7 and 0.75 pass different samples", + &eval_api.EvaluatorSummary{Definition: threshold(0.75)}, + "0.75", + }, + { + "a trailing zero is kept so the column stays aligned", + &eval_api.EvaluatorSummary{Definition: threshold(0.8)}, + "0.80", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, evaluatorPassThreshold(tc.in)) + }) + } +} + +func TestRenderEvaluatorVersionsShowsTheThreshold(t *testing.T) { + raised, held := 0.80, 0.70 + var buf bytes.Buffer + + cmd := &cobra.Command{} + cmd.SetOut(&buf) + + require.NoError(t, renderEvaluatorVersions(cmd, &eval_api.EvaluatorListResponse{ + Value: []eval_api.EvaluatorSummary{ + { + Version: "3", + CreatedAt: "2026-08-03T11:22:04Z", + Description: "Raised threshold, split cites_policy", + Definition: &eval_api.EvaluatorContract{PassThreshold: &raised}, + }, + { + Version: "2", + CreatedAt: "2026-08-01T14:07:33Z", + Description: "Tightened offers_next_step criteria", + Definition: &eval_api.EvaluatorContract{PassThreshold: &held}, + }, + }, + })) + + out := buf.String() + for _, want := range []string{ + "VERSION", "CREATED AT", "PASS THRESHOLD", "DESCRIPTION", + "0.80", "0.70", + "Raised threshold, split cites_policy", + } { + assert.Contains(t, out, want) + } + + // Name and type are constant down the listing, so printing them would cost + // width and say nothing. + assert.NotContains(t, out, "NAME") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go new file mode 100644 index 00000000000..4db7c6da2a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ids are per declaration. A shared key works only while a config has one +// group: with two, the second deploy finds the first's id cached, confirms it +// exists, and hands it back for the wrong group — so group A silently scores +// group B's criteria. +func TestIDKey_IsPerName(t *testing.T) { + a := idKey("eval", "quality-a") + b := idKey("eval", "quality-b") + + assert.NotEqual(t, a, b, "two groups must not share an id key") + assert.Contains(t, a, "QUALITY_A") + assert.True(t, len(a) > 3 && a[len(a)-3:] == "_ID") +} + +// Names that are not valid env identifiers still have to produce distinct, +// stable keys. +// +// The readable half of the key cannot tell "my group" from "my-group" — both +// sanitize to MY_GROUP. Letting them share a key is the collision the test +// above describes: the second declaration finds the first's id cached and +// scores the wrong group. +func TestIDKey_NormalizesNames(t *testing.T) { + assert.NotEqual(t, idKey("eval", "my group"), idKey("eval", "my-group"), + "names that sanitize alike are still different names") + assert.NotEqual(t, idKey("eval", "a"), idKey("dataset", "a"), + "the kind keeps different resources apart") + + assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my group"), + "the same name must key the same way on every deploy") +} + +// The id and version keys for the same declaration must not collide. +func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { + assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) +} + +// The shared EVAL_ID entry is written by every deploy, so it cannot say which +// declaration it belongs to. Reading it for a config that names a single eval +// let a file whose one entry had been replaced run the previous eval's criteria +// over the new one's rows, reported as success. An eval's id is read from the +// entry recorded under its own name and nowhere else. +func TestEvalIDIsReadFromTheEvalsOwnEntry(t *testing.T) { + assert.NotEqual(t, envKeyEvalID, idKey("eval", "quality")) + assert.NotEqual(t, idKey("eval", "quality"), idKey("eval", "nightly"), + "two declarations cannot share an entry") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go new file mode 100644 index 00000000000..0640c14ff4f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary. Warning about nowhere to persist would +// be noise on every standalone invocation. +func TestNoAzdEnvironmentIsRecognizable(t *testing.T) { + err := fmt.Errorf("%w to write %s into", errNoAzdEnvironment, "EVAL_RUN_ID") + + require.ErrorIs(t, err, errNoAzdEnvironment, + "callers rely on telling this apart from a failed write") + require.Contains(t, err.Error(), "EVAL_RUN_ID", + "the key is still named when the message is shown") +} + +// A write that fails for any other reason stays reportable. +func TestOtherEnvironmentFailuresStayReportable(t *testing.T) { + err := fmt.Errorf("writing %s to the azd environment: %w", "EVAL_RUN_ID", errors.New("rpc failed")) + + require.NotErrorIs(t, err, errNoAzdEnvironment) + require.Contains(t, err.Error(), "rpc failed") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go new file mode 100644 index 00000000000..d05445f56f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// chooseEvalIn is chooseEval for the run commands, which hold a directory +// rather than a loaded configuration. A configuration that will not open is +// left to the command that opens it properly, so the error stays the same one. +func chooseEvalIn(cmd *cobra.Command, evalDir, named string) string { + if named != "" || noPrompt(cmd) { + return named + } + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return named + } + return chooseEval(cmd, cfg, named) +} + +// chooseEval settles which eval a command means when the caller named none. +// +// Refusing is right under --no-prompt, where there is nobody to ask. Standing +// at a terminal it is not: the command holds the whole candidate list, and the +// documented scenarios declare a second eval, so every bare `run start` after +// that would fail permanently. +// +// Returning the name empty leaves the existing error to the caller, which is +// what happens whenever the prompt cannot run. +func chooseEval(cmd *cobra.Command, cfg *project.EvalConfig, named string) string { + if named != "" || cfg == nil || len(cfg.Evals) < 2 || noPrompt(cmd) { + return named + } + + azdClient, err := azdext.NewAzdClient() + if err != nil { + return named + } + defer azdClient.Close() + + names := cfg.EvalNames() + choices := make([]*azdext.SelectChoice, 0, len(names)) + for i := range names { + choices = append(choices, &azdext.SelectChoice{Label: names[i], Value: names[i]}) + } + + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectEvalPrompt(), + Choices: choices, + }, + }) + if err != nil { + return named + } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue -- indistinguishable from the first choice. Reading it as a + // selection would start a billed run against an eval nobody picked. + if resp == nil || resp.Value == nil { + return named + } + index := int(resp.GetValue()) + if index < 0 || index >= len(names) { + return named + } + return names[index] +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go new file mode 100644 index 00000000000..d17840b398c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// configWith builds a configuration declaring the named evals. +func configWith(names ...string) *project.EvalConfig { + cfg := &project.EvalConfig{} + for _, n := range names { + cfg.Evals = append(cfg.Evals, project.Eval{Name: n}) + } + return cfg +} + +// chooseEval only asks when asking can settle something. Every case below +// resolves without a prompt, so none of them reaches the azd client. +func TestChooseEvalOnlyAsksWhenThereIsAChoice(t *testing.T) { + t.Run("a name given is never second-guessed", func(t *testing.T) { + got := chooseEval(newEvalCreateCommand(), configWith("a", "b"), "b") + assert.Equal(t, "b", got) + }) + + t.Run("one declared eval needs no question", func(t *testing.T) { + got := chooseEval(newEvalCreateCommand(), configWith("only"), "") + assert.Empty(t, got, "the caller resolves the single eval, so nothing is chosen here") + }) + + t.Run("no configuration is left to the caller", func(t *testing.T) { + assert.Empty(t, chooseEval(newEvalCreateCommand(), nil, "")) + }) + + t.Run("--no-prompt keeps the error", func(t *testing.T) { + cmd := newEvalCreateCommand() + cmd.Flags().Bool("no-prompt", true, "") + + got := chooseEval(cmd, configWith("a", "b"), "") + + assert.Empty(t, got, + "there is nobody to ask, so the command must still refuse rather than guess") + }) +} + +// Returning the name unchanged is what leaves the existing error in place, and +// that error is the one users praised: it counts the evals and names them all. +func TestSeveralEvalsErrorStillNamesEveryCandidate(t *testing.T) { + cfg := configWith("obs-trace-eval", "obs-eval") + cmd := newEvalCreateCommand() + // Without this the picker reaches azdext.NewAzdClient and attempts a real + // RPC, which passes only because resolving an empty address fails fast. + // This test is about the message, not about network behaviour. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := cfg.Eval(chooseEval(cmd, cfg, "")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "obs-trace-eval") + assert.Contains(t, err.Error(), "obs-eval") + assert.Contains(t, err.Error(), "--eval") +} + +// A directory with no configuration must not turn into a prompt, and must not +// swallow the error the command that opens it properly will raise. +func TestChooseEvalInLeavesAnAbsentConfigAlone(t *testing.T) { + assert.Empty(t, chooseEvalIn(newEvalCreateCommand(), t.TempDir(), "")) + assert.Equal(t, "named", chooseEvalIn(newEvalCreateCommand(), t.TempDir(), "named")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go new file mode 100644 index 00000000000..e6c7b5ef565 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Creation normally belongs to `azd up`, which owns reconciliation. `create` +// is the same path for a single eval outside a project, and takes the +// configuration rather than a wall of flags so there is never a second +// definition to maintain. + +// newEvalCreateCommand creates one declared eval without deploying the rest. +func newEvalCreateCommand() *cobra.Command { + var ( + fromFile string + evalDir string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create one eval declared in the configuration.", + Long: "Create one eval declared in the configuration.\n\n" + + "`azd up` reconciles every eval in the file. This creates a single one, " + + "for a project that is not deployed as a whole — or, with --from-file, " + + "for no project at all.\n\n" + + "The name is optional while the configuration declares exactly one eval.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + path := fromFile + if path == "" { + dir, err := resolveEvalDir(ctx, evalDir) + if err != nil { + return err + } + if path, err = project.ResolveEvalConfigPath(dir); err != nil { + return err + } + } + cfg, err := project.LoadEvalConfig(path) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + eval, err := cfg.Eval(chooseEval(cmd, cfg, firstArg(args))) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Local sources resolve against the file, not the working directory, + // so the columns are read from where the declaration points. + baseDir := filepath.Dir(path) + datasetPath := "" + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + datasetPath = project.ResolveSource(baseDir, decl.Source) + } + + reconciler := &evalReconciler{ec: ec} + out := cmd.OutOrStdout() + + // Before anything is pushed. Publishing is not free -- a dataset + // version is immutable and the number climbs on every attempt -- so + // a declaration the evaluators cannot satisfy is refused first. + if err := checkEvaluatorRequirements(eval, ec.evaluatorSchemas(ctx)); err != nil { + return err + } + // Reported per artifact, because "publishes nothing when nothing + // changed" is the contract a reader is checking here and a single + // closing line cannot show it. Silent under -o json. + say := func(kind, name, version string, changed bool) { + if isJSON(cmd) { + return + } + if changed { + fmt.Fprintln(out, messages.PublishedVersion(kind, name, version)) + } else { + fmt.Fprintln(out, messages.UnchangedAtVersion(kind, name, version)) + } + } + + // The eval names its dataset and evaluators, and the service resolves + // those names when the eval is created, so they have to be published + // first. `azd up` reconciles the whole file; this reconciles only what + // this eval refers to, which is also what makes a rubric edit reach + // the service without a full deploy. + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + version, changed, err := reconciler.EnsureDataset(ctx, *decl, datasetPath) + if err != nil { + return messages.DatasetProblem(decl.Name, err) + } + say("dataset", decl.Name, version, changed) + } + for _, ref := range eval.Evaluators { + decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) + // A built-in, or one already registered, has nothing local to publish. + if !ok || decl.Source == "" { + continue + } + local := project.ResolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureEvaluator(ctx, *decl, local) + if err != nil { + return messages.EvaluatorProblem(decl.Name, err) + } + say("evaluator", decl.Name, version, changed) + } + + id, created, err := reconciler.EnsureEval(ctx, *eval, datasetPath) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": id, "name": eval.Name, + }) + } + if created { + fmt.Fprint(out, messages.EvalCreated(eval.Name, id)) + } else { + fmt.Fprint(out, messages.EvalUnchanged(eval.Name, id)) + } + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Read the configuration from this path instead of the eval directory.") + cmd.Flags().StringVar(&evalDir, "path", "", + "Directory holding the evaluation configuration. Defaults to the directory "+ + "`init` scaffolded, otherwise ./evals.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalListCommand() *cobra.Command { + var ( + limit int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evals.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) + if err != nil { + return messages.ListingEvals(err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Data) + } + if len(list.Data) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvals()) + return nil + } + rows := make([][]string, 0, len(list.Data)) + for _, e := range list.Data { + rows = append(rows, []string{e.ID, e.Name}) + } + return emitTable(cmd.OutOrStdout(), []string{"EVAL ID", "NAME"}, rows) + }, + } + + cmd.Flags().IntVar(&limit, "limit", 0, "Cap the number of evals returned.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an eval definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // The argument reads as an id. `list` reports names, and this + // refused the very name it points the reader at, so a name is + // resolved before giving up. + if resolved := ec.evalIDNamed(ctx, evalID); resolved != "" { + group, err = ec.evalClient.GetOpenAIEval(ctx, resolved) + } + } + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvalNotFound(evalID) + } + return messages.ReadingEval(evalID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), group) + } + detail := []field{ + {"Id", group.ID}, + {"Name", group.Name}, + // CreatedAt is `any` because the service sends epoch seconds here + // and RFC3339 elsewhere; fmt.Sprint on the former prints a float + // in scientific notation. + {"Created", timestampString(group.CreatedAt)}, + {"Created By", group.CreatedBy}, + } + // Without this the command answers "does this id exist", which is + // not what a definition is, nor what its own help promises. + if graders := evalGraders(group); graders != "" { + detail = append(detail, field{"Evaluators", graders}) + } + return emitDetail(cmd.OutOrStdout(), detail) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// evalGraders lists the evaluators the eval grades with, preferring the +// reference a caller would recognize over the criterion label. +// +// data_source_config is deliberately not shown beside it: every eval this +// extension creates carries type "custom", which describes the item schema +// rather than where the rows come from, so a "Source" row would read as an +// answer while always saying the same thing. +func evalGraders(group *eval_api.OpenAIEval) string { + if group == nil { + return "" + } + names := make([]string, 0, len(group.TestingCriteria)) + for _, c := range group.TestingCriteria { + name := c.EvaluatorName + if name == "" { + name = c.Name + } + if name == "" { + continue + } + if c.EvaluatorVersion != "" { + name += " (" + c.EvaluatorVersion + ")" + } + names = append(names, name) + } + return strings.Join(names, ", ") +} + +func newEvalDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an eval and everything under it.", + Long: "Delete an eval and everything under it.\n\n" + + "An eval owns its runs, so deleting one discards their results too.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // `list` reports names, so a name is what a reader has to hand. + // An eval is immutable, though, so editing a declaration leaves + // another under the same name, and this deletes the runs under + // whichever it picks: with more than one it asks rather than guesses. + ids, listErr := ec.evalIDsNamed(ctx, evalID) + if listErr != nil { + // Reporting the eval gone on a listing we could not + // read would be a delete silently doing nothing. + return listErr + } + switch len(ids) { + case 0: + case 1: + evalID = ids[0] + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + default: + return messages.AmbiguousEvalName(evalID, ids) + } + } + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvalGone(evalID) + } + return messages.DeletingEval(evalID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": evalID, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.EvalDeleted(evalID)) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go new file mode 100644 index 00000000000..b44d34be590 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// EVAL_ID is written by every deploy, so nothing tells a value meant for this +// declaration from one left behind by the eval it replaced. +// +// Reading it as a fallback meant a file whose one entry had been swapped ran +// the previous eval's criteria over the new one's rows and reported success, +// and `run cancel` with no arguments cancelled a run of the eval the file no +// longer described -- a destructive verb on a resource picked by accident. +// +// This is the only enforcement of that. The reasoning lives in a comment on +// the reconciler and in another on `resolveEvalID`, and a comment cannot fail. +func TestRecordedEvalIDIgnoresTheSharedKey(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + envKeyEvalID: "evalgroup_the_one_this_replaced", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + + assert.Empty(t, ec.recordedEvalID(context.Background(), "nightly"), + "a shared key cannot say which declaration it belongs to") + + // The entry recorded under the eval's own name does answer, which is what + // makes the miss above a deliberate refusal rather than a broken read. + env.values[idKey("eval", "nightly")] = "evalgroup_nightly" + require.Equal(t, "evalgroup_nightly", ec.recordedEvalID(context.Background(), "nightly")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go new file mode 100644 index 00000000000..fbe5267fb61 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" +) + +// `show` is documented as showing an eval definition, and answered with the id, +// the name and who created it -- true of any eval, and none of the definition. +// The graders are the part of the definition the service does return. +func TestShowSurfacesWhatTheEvalGrades(t *testing.T) { + group := &eval_api.OpenAIEval{ + ID: "eval_1", + Name: "support-trace-eval", + TestingCriteria: []eval_api.TestingCriterion{ + {Name: "task_adherence", EvaluatorName: "builtin.task_adherence"}, + {Name: "quality", EvaluatorName: "support-quality", EvaluatorVersion: "2"}, + }, + } + + assert.Equal(t, "builtin.task_adherence, support-quality (2)", evalGraders(group)) +} + +// The criterion label is what the service echoes when no evaluator reference +// was recorded, so it is better than printing nothing. +func TestShowFallsBackToTheCriterionLabel(t *testing.T) { + group := &eval_api.OpenAIEval{ + TestingCriteria: []eval_api.TestingCriterion{{Name: "custom-grader"}}, + } + + assert.Equal(t, "custom-grader", evalGraders(group)) +} + +// An older eval, or one the service answers without a definition, still shows +// its identity rather than blank rows. +func TestShowOmitsWhatTheServiceDidNotSend(t *testing.T) { + assert.Empty(t, evalGraders(&eval_api.OpenAIEval{ID: "eval_1"})) + assert.Empty(t, evalGraders(nil)) + + // A criterion carrying no name at all contributes nothing rather than an + // empty entry with a stray separator. + assert.Empty(t, evalGraders(&eval_api.OpenAIEval{ + TestingCriteria: []eval_api.TestingCriterion{{Type: "azure_ai_evaluator"}}, + })) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go new file mode 100644 index 00000000000..c7527a1f73a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -0,0 +1,341 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Where the configuration lives is settled by one cascade -- --path, then the +// path `init` recorded in the azd environment, then ./evals -- and these tests +// pin it. +// +// `init --path ./quality` wrote a configuration that `run` then looked for +// under ./evals and reported as missing, while azure.yaml's $ref pointed at it +// correctly the whole time. The path init used is remembered so the flag does +// not have to be repeated on every later command. +// +// That fix reached `run` and stopped there. In a project scaffolded outside +// ./evals, `create` went on reporting the configuration missing and `generate` +// went on submitting a billed job and writing a *second* configuration under +// ./evals that nothing else read. So these tests are written over every +// command, not over the one that was wrong at the time. + +func TestEvalDirCascade(t *testing.T) { + // No azd environment: there is nothing to read, so only flag and default apply. + ec := &evalContext{} + + dir, err := ec.evalDir(context.Background(), "") + require.NoError(t, err) + assert.Equal(t, project.DefaultEvalDir, dir, "nothing given anywhere is ./evals") + + dir, err = ec.evalDir(context.Background(), "quality") + require.NoError(t, err) + assert.Equal(t, "quality", dir, "--path wins") +} + +func TestEvalDirCascadeAnswersInOrder(t *testing.T) { + cases := []struct { + name string + flag string + recorded string + want string + }{ + { + name: "the flag wins", + flag: "./given", + recorded: "./recorded", + want: "./given", + }, + { + name: "the flag wins even over nothing recorded", + flag: "./given", + recorded: "", + want: "./given", + }, + { + name: "what init recorded is used when no flag was given", + flag: "", + recorded: "./quality", + want: "./quality", + }, + { + name: "the default is the last resort", + flag: "", + recorded: "", + want: project.DefaultEvalDir, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := evalDirCascade(tc.flag, func() (string, error) { + return tc.recorded, nil + }) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// A read that failed is not a project that recorded nothing. Defaulting on it +// is how `generate` would write a second configuration under ./evals for a +// reason nobody could reproduce, so the failure has to come back out. +func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) { + boom := errors.New("the environment could not be read") + + got, err := evalDirCascade("", func() (string, error) { return "", boom }) + + require.ErrorIs(t, err, boom) + assert.Empty(t, got, "a failed read must not answer with the default") +} + +// A --path that was given is the answer on its own, so a broken azd cannot +// stop a caller who already said where to look. +func TestEvalDirCascadeIgnoresAFailedReadWhenPathWasGiven(t *testing.T) { + got, err := evalDirCascade("./given", func() (string, error) { + return "", errors.New("the environment could not be read") + }) + + require.NoError(t, err) + assert.Equal(t, "./given", got) +} + +// Each read is a round trip, and a --path that was given makes it unnecessary. +func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { + var asked int + got, err := evalDirCascade("", func() (string, error) { + asked++ + return "", nil + }) + + require.NoError(t, err) + assert.Equal(t, project.DefaultEvalDir, got) + assert.Equal(t, 1, asked, "the recorded path should be read exactly once") + + asked = 0 + _, err = evalDirCascade("./given", func() (string, error) { + asked++ + return "", nil + }) + require.NoError(t, err) + assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip") +} + +// --path defaults to empty, not to ./evals, so "not given" stays +// distinguishable from "given the default". A non-empty default shadows the +// path init recorded, because level 1 only yields on an empty value -- so the +// command has opted out of the cascade without saying so. +// +// This was asserted for `run start` alone, which is exactly how `create`, +// `generate` and `init` came to be filling the default in. It is written over +// the whole tree now. +func TestPathFlagsLeaveRoomForTheRecordedPath(t *testing.T) { + var checked int + walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { + f := cmd.Flags().Lookup("path") + if f == nil { + return + } + checked++ + assert.Empty(t, f.DefValue, + "`azd ai eval %s --path` defaults to %q, so the path `init` recorded can "+ + "never be reached: level 1 of the cascade only yields on an empty value", + name, f.DefValue) + }) + + // If --path is ever renamed, the loop above passes by visiting nothing. + assert.GreaterOrEqual(t, checked, 3, + "expected --path on at least init, generate and eval create; found %d", checked) +} + +// Every command that reads the configuration has to be able to say where it is, +// or a project scaffolded with --path is unreachable from that command. +// +// `create` was missing from this list, and was one of the two commands that +// could not find a configuration outside ./evals. +func TestCommandsReadingTheConfigTakePath(t *testing.T) { + for _, path := range []string{"run start", "init", "generate", "create"} { + cmd := find(t, path) + assert.NotNilf(t, cmd.Flags().Lookup("path"), + "%s reads the configuration, so it must accept --path", path) + } +} + +// Guards against the tree walk above passing because the flag was renamed. +func TestPathFlagIsStillCalledPath(t *testing.T) { + var names []string + walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name == "path" { + names = append(names, name) + } + }) + }) + + for _, want := range []string{"init", "generate", "create", "run start"} { + assert.Contains(t, names, want) + } +} + +// The recorded key is what `init` writes and what the other commands read; a +// rename on one side alone silently stops the hand-off working. +func TestEvalPathEnvKey(t *testing.T) { + assert.Equal(t, "EVAL_CONFIG_PATH", envKeyEvalPath) +} + +// `init` prints the commands to run next, and the claim those lines make is +// that they run as printed. A scaffold written somewhere other than ./evals is +// only reachable by a command that names it, because EVAL_CONFIG_PATH is +// recorded best effort and `init` succeeds without an azd environment to record +// it in. +func TestNextStepsRunAsPrinted(t *testing.T) { + cases := []struct { + name string + evalDir string + deployCmd string + wantPath bool + }{ + { + name: "a scaffold outside ./evals names itself", + evalDir: "./quality", + deployCmd: "azd ai eval create", + wantPath: true, + }, + { + name: "the default directory needs no flag", + evalDir: project.DefaultEvalDir, + deployCmd: "azd ai eval create", + wantPath: false, + }, + { + name: "an unrecorded directory needs no flag", + evalDir: "", + deployCmd: "azd ai eval create", + wantPath: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: tc.evalDir} + for _, step := range s.nextSteps(tc.deployCmd) { + assert.Equal(t, tc.wantPath, strings.Contains(step, "--path "), + "step %q", step) + } + }) + } +} + +// `azd up` provisions and then deploys, reading azure.yaml -- which already +// $refs the configuration wherever it was written. It takes none of this +// extension's flags, so handing it --path prints a step that fails. +func TestNextStepsNeverFlagAzdUp(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: "./quality"} + + steps := s.nextSteps(azdUpCommand) + assert.Contains(t, steps, azdUpCommand, + "`azd up` should be suggested exactly as it is run") + for _, step := range steps { + if strings.HasPrefix(step, azdUpCommand) { + assert.NotContains(t, step, "--path", "step %q", step) + } + } +} + +// A generated next step already carried --target and --generation-model for the +// same reason. --path joins them. +func TestGenerateStepNamesTheScaffoldedDirectory(t *testing.T) { + s := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + evalDir: "./quality", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + rubricName: "support-agent-quality", + datasetName: "support-agent-dataset", + generateDataset: true, + generateRubric: true, + } + + steps := s.nextSteps("azd ai eval create") + if assert.Len(t, steps, 1) { + for _, want := range []string{ + "--target support-agent", + "--generation-model gpt-4.1-nano", + "--path ./quality", + } { + assert.Contains(t, steps[0], want) + } + } +} + +// A directory with a space in it printed `--path ./team evals`, which resolves +// ./team and reports the configuration missing -- the printed step failing in +// the one case it was added for. Found by running it, not by reading it. +func TestNextStepQuotesADirectoryThatNeedsIt(t *testing.T) { + cases := []struct { + name string + evalDir string + want string + }{ + { + name: "a space", + evalDir: "./team evals", + want: `--path "./team evals"`, + }, + { + name: "a windows path with a space", + evalDir: `C:\Users\Me\My Evals`, + want: `--path "C:\Users\Me\My Evals"`, + }, + { + name: "a plain relative path is left alone", + evalDir: "./quality", + want: "--path ./quality", + }, + { + name: "a plain windows path is left alone", + evalDir: `C:\Users\Me\quality`, + want: `--path C:\Users\Me\quality`, + }, + { + name: "a character the shell would expand", + evalDir: "./eval$dir", + want: `--path "./eval$dir"`, + }, + { + name: "a character that would end the command", + evalDir: "./a;rm -rf b", + want: `--path "./a;rm -rf b"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: tc.evalDir} + steps := s.nextSteps("azd ai eval create") + require.NotEmpty(t, steps) + for _, step := range steps { + assert.Contains(t, step, tc.want, "step %q", step) + } + }) + } +} + +// Backslashes must survive: doubling them is right for bash and wrong for the +// two shells most likely to be reading a path that looks like this. +func TestQuoteForShellLeavesBackslashesAlone(t *testing.T) { + assert.Equal(t, `"C:\Users\Me\My Evals"`, quoteForShell(`C:\Users\Me\My Evals`)) + assert.Equal(t, `C:\Users\Me\Evals`, quoteForShell(`C:\Users\Me\Evals`)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go new file mode 100644 index 00000000000..b41ba537fe3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "sort" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" +) + +// evalRef is what `--eval` resolved to: the service id, plus the declaration +// behind it when there was one. +// +// Commands need both. The id is what every route under {eval_id} takes, and the +// declaration is what says which dataset and target a new run should use. +type evalRef struct { + ID string + Eval *project.Eval + Config *project.EvalConfig + ConfigPath string +} + +// Declared reports whether the reference came from the configuration. +func (r evalRef) Declared() bool { return r.Eval != nil } + +// resolveEvalRef turns `--eval` into an id. +// +// One flag takes a name or an id, matching `azd ai training job show`, whose +// --name is documented as "Job name/ID". Name is tried first, in three cases: +// a name in evals: with a recorded id resolves to it; a name in evals: with +// none fails fast naming `azd up` rather than returning a service 404; and +// anything else is sent as an id. +// +// Ids matter because an eval created by `azd ai eval create` has no evals: +// entry, and because the environment records one id per name, so editing a +// declaration leaves every run of the previous eval reachable only by id. +func (ec *evalContext) resolveEvalRef( + ctx context.Context, + evalDir, nameOrID string, +) (evalRef, error) { + configPath, err := project.ResolveEvalConfigPath(evalDir) + if err != nil { + return evalRef{}, err + } + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return evalRef{}, err + } + + if cfg != nil { + if err := cfg.ValidateForLookup(); err != nil { + return evalRef{}, err + } + eval, err := cfg.Eval(nameOrID) + switch { + case err == nil: + id := ec.recordedEvalID(ctx, eval.Name) + if id == "" { + // Nothing recorded is not the same as nothing published. The + // id is kept in the azd environment, so a run against + // --project-endpoint with no project, or a `create` that ran + // before the environment existed, leaves a published eval + // with no note of it. The service lists evals by name, so + // ask it before deciding this was never deployed. + // + // Refused rather than guessed when a name carries several, as + // `eval delete` already does. Newest-wins is fine for showing + // something, but this id also reaches `run start`, and grading + // against the wrong definition produces results that look + // right and answer a different question. + ids, err := ec.evalIDsNamed(ctx, eval.Name) + if err != nil { + // A listing that failed is not a listing that came + // back empty. Falling through to "not deployed" + // sends the reader to republish an eval that + // already exists -- which is what a listing failing + // under concurrent `run start` actually produced. + return evalRef{}, err + } + if len(ids) > 1 { + return evalRef{}, messages.AmbiguousEvalName(eval.Name, ids) + } + if len(ids) == 1 { + id = ids[0] + } + } + if id == "" { + // With no environment there was nowhere an id could have been + // recorded, so telling the reader to deploy again would not + // help. Only said when azd confirmed there is none. + if ec.confirmedNoAzdEnvironment(ctx) { + return evalRef{}, messages.NoEnvironmentToRememberEval(eval.Name) + } + // That call recovers the environment name when the first + // lookup missed it -- a transient failure in newEvalContext + // leaves it empty, and recordedEvalID answers "" without + // asking when it is. Now that there is a name, ask properly + // before reporting a deployed eval as missing. + if id = ec.recordedEvalID(ctx, eval.Name); id == "" { + return evalRef{}, messages.EvalNotDeployedYet( + eval.Name, ec.deployCommand(ctx)) + } + } + return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil + case nameOrID == "": + // No name to fall back on, so the configuration's own complaint — + // none declared, or several to choose between — is the answer. + return evalRef{}, err + } + } + + if nameOrID == "" { + return evalRef{}, messages.NoEvalNamedOrDeclared(configPath) + } + // Not a declared name, so it is an id. + return evalRef{ID: nameOrID}, nil +} + +// recordedEvalID reads the id `azd up` stored for a declared eval. +// +// Only the entry recorded under this eval's own name. EVAL_ID is written by +// every deploy as well as being settable by hand, so nothing tells a value that +// was meant for this declaration from one left behind by the eval it replaced. +// Reading it for a config that names a single eval meant a file whose one entry +// had been swapped for a different one ran the previous eval's criteria over +// the new one's rows, reported as success. A miss falls through to the service +// listing by name, which answers the question the id was standing in for. +func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { + return ec.getEnvValue(ctx, idKey("eval", evalName)) +} + +// evalIDNamed finds the id of the eval the service lists under this name. +// +// Evals are addressed by id, but every listing reports a name, so a name is +// what a reader has to hand. Returns empty when nothing matches, leaving the +// caller's not-found reporting alone. +// +// The newest match wins when a name is carried by several. An eval is +// immutable, so editing a declaration creates another one under the same name, +// and the newest is the one the configuration currently describes. +// +// A listing that failed answers empty here, unlike in resolveEvalRef: both +// callers reach this only after the service already returned 404 for the id +// they were given, and that refusal is what they report. +func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { + ids, err := ec.evalIDsNamed(ctx, name) + if err != nil || len(ids) == 0 { + return "" + } + return ids[0] +} + +// evalIDsNamed finds every eval the service lists under this name, newest +// first, so a caller that must not guess can see the ambiguity. +// +// The order is established here rather than taken from the service, which does +// not promise one. timestampString normalizes both shapes the service uses for +// created_at to RFC3339 UTC, and those sort chronologically as text. An eval +// whose timestamp is missing or unparseable sorts last rather than winning by +// accident. +func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) ([]string, error) { + list, err := ec.evalClient.ListOpenAIEvals(ctx, 0) + if err != nil { + return nil, messages.ListingEvals(err) + } + if list == nil { + return nil, nil + } + return idsNamedIn(list, name), nil +} + +// idsNamedIn picks the evals carrying this name, newest first. +// +// timestampString normalizes both shapes the service uses for created_at to +// RFC3339 UTC, and those sort chronologically as text. An eval whose timestamp +// is missing or unparseable sorts last rather than winning by accident. +func idsNamedIn(list *eval_api.OpenAIEvalList, name string) []string { + var matches []eval_api.OpenAIEval + for _, e := range list.Data { + if e.Name == name { + matches = append(matches, e) + } + } + sort.SliceStable(matches, func(i, j int) bool { + return timestampString(matches[i].CreatedAt) > timestampString(matches[j].CreatedAt) + }) + ids := make([]string, 0, len(matches)) + for _, m := range matches { + ids = append(ids, m.ID) + } + return ids +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go new file mode 100644 index 00000000000..719a58f1c17 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -0,0 +1,329 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// reconcilerListingVersions builds a reconciler whose dataset client answers a +// version listing with these versions. +func reconcilerListingVersions(t *testing.T, versions ...string) *evalReconciler { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + values := make([]map[string]any, 0, len(versions)) + for _, v := range versions { + values = append(values, map[string]any{"name": "golden", "version": v}) + } + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalReconciler{ec: &evalContext{ + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} +} + +// A version published outside the repo would otherwise be overwritten by the +// next deploy. The evaluator side of this is guarded; the dataset side is the +// same risk against content nobody has a copy of. +func TestCheckDatasetDriftRefusesANewerPublishedVersion(t *testing.T) { + r := reconcilerListingVersions(t, "1.0", "2.0", "3.0") + + err := r.checkDatasetDrift(context.Background(), "golden", "2.0") + + require.Error(t, err) + assert.Contains(t, err.Error(), "3.0", "the version that is actually there") + assert.Contains(t, err.Error(), "2.0", "and the one this repo last deployed") + assert.Contains(t, err.Error(), "outside this configuration") + assert.Contains(t, err.Error(), "version: 3.0", "the fix is a pin the user can paste") +} + +// Matching versions are the ordinary case and must stay silent, or every +// deploy would report drift. +func TestCheckDatasetDriftAcceptsAMatch(t *testing.T) { + r := reconcilerListingVersions(t, "1.0", "2.0") + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) +} + +// The listing is eventually consistent and answers with nothing for a second +// or two after a publish. Reading that as "the project is behind" would report +// drift on a dataset this repo had just deployed. +func TestCheckDatasetDriftIgnoresAnEmptyListing(t *testing.T) { + r := reconcilerListingVersions(t) + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) + latest, err := r.latestDatasetVersion(context.Background(), "golden") + require.NoError(t, err) + assert.Empty(t, latest) +} + +// Only a newer version is someone else's work. An older one means this repo is +// ahead, which the deploy is about to fix anyway. +func TestCheckDatasetDriftIgnoresAnOlderVersion(t *testing.T) { + r := reconcilerListingVersions(t, "1.0") + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0"), + "a project behind this repo is not drift") +} + +// A listing that failed is not evidence there was no newer version. The drift +// check exists to catch someone else's publish, so it fails closed: tolerating +// the error let a 403 or a timeout skip the guard silently. +func TestLatestDatasetVersionSurfacesAFailedListing(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + r := &evalReconciler{ec: &evalContext{ + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} + + _, err := r.latestDatasetVersion(context.Background(), "golden") + require.Error(t, err, "a listing we could not read is not proof there was no newer version") + require.Error(t, r.checkDatasetDrift(context.Background(), "golden", "2.0"), + "the drift guard fails closed rather than skipping silently") +} + +// writeEvalYAML puts a configuration in a temp dir and returns the dir. +func writeEvalYAML(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + evals := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(evals, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(evals, "eval.yaml"), []byte(body), 0o600)) + return evals +} + +// evalContextListingEvals builds a context whose service lists exactly these +// evals. resolveEvalRef asks the service by name when no id was recorded, so a +// context without a client cannot exercise it. +func evalContextListingEvals(t *testing.T, envName, body string) *evalContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalContext{ + envName: envName, + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + } +} + +// evalContextRefusingToListEvals builds a context whose service will not answer +// the listing at all, which is a different thing from listing nothing. +func evalContextRefusingToListEvals(t *testing.T, envName string, status int) *evalContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":{"code":"TooManyRequests"}}`)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalContext{ + envName: envName, + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + } +} + +// A listing that failed is not a listing that came back empty. Both leave no +// id, and only one of them means the eval was never deployed. +// +// This is not hypothetical: four concurrent `run start` calls against a live +// project produced exactly one of these, and the reader was told a deployed +// eval did not exist and to run `azd up` -- which would publish a second copy +// of something already there. +func TestResolveEvalRefReportsARefusedListingRatherThanCallingItUndeployed(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextRefusingToListEvals(t, "dev", http.StatusTooManyRequests) + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "listing evals", + "the failure that actually happened is the one reported") + assert.NotContains(t, err.Error(), "azd up", + "deploying again does not fix a listing the service refused") +} + +// A declared eval that was never deployed has no id to address, and the +// service would answer 404 for a name it never saw. Naming the command that +// deploys is the difference between a dead end and a next step. +func TestResolveEvalRefFailsFastOnAnUndeployedDeclaration(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + // An environment exists; the id was simply never recorded in it. Without + // this the case under test is "nowhere to record", which is a different + // answer. The service lists nothing, which is what never deployed looks + // like from the outside. + ec := evalContextListingEvals(t, "dev", `{"data":[]}`) + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + // No azd project stands behind this context, so there is no infrastructure + // to provision and neither `azd up` nor `azd deploy` would run here. + assert.Contains(t, err.Error(), "azd ai eval create", + "the error has to say what would fix it") +} + +// The id lives in the azd environment, so `--project-endpoint` against a +// directory that never had one has a published eval and no note of it. Failing +// there would make the declaration unusable outside a project, though the +// service can be asked for the same name. +func TestResolveEvalRefFindsAPublishedEvalByName(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextListingEvals(t, "", + `{"data":[{"id":"eval_published","name":"support-quality"}]}`) + + ref, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.NoError(t, err) + assert.Equal(t, "eval_published", ref.ID, + "the service knows the id this environment never recorded") + assert.True(t, ref.Declared(), "and it is still the declaration that was matched") +} + +// With no azd environment at all there is nowhere the id could have been +// recorded, so `create` may well have published this eval and had nowhere to +// note it. Sending the reader to `azd up` lands them in the same place. +func TestResolveEvalRefNamesTheMissingEnvironment(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextListingEvals(t, "", `{"data":[]}`) + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd env new", + "the fix is an environment, not another deploy") + assert.NotContains(t, err.Error(), "azd up", + "deploying again would record the id in the same nowhere") +} + +// An eval made by `azd ai eval create` has no evals: entry, so anything that +// is not a declared name is sent on as an id rather than refused. +func TestResolveEvalRefTreatsAnUnknownNameAsAnID(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := &evalContext{} + + ref, err := ec.resolveEvalRef(context.Background(), dir, "eval_68a1f2c3") + + require.NoError(t, err) + assert.Equal(t, "eval_68a1f2c3", ref.ID) + assert.False(t, ref.Declared(), "an id carries no declaration to run from") + assert.Nil(t, ref.Eval) +} + +// With no configuration and no name there is nothing to resolve, and the +// message has to say where a declaration would have been looked for. +func TestResolveEvalRefWithoutAConfigurationOrAName(t *testing.T) { + ec := &evalContext{} + + _, err := ec.resolveEvalRef(context.Background(), t.TempDir(), "") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--eval") + assert.Contains(t, err.Error(), "eval.yaml") +} + +// Outside a project an id is still enough to address every route under it. +func TestResolveEvalRefAcceptsAnIDWithoutAConfiguration(t *testing.T) { + ec := &evalContext{} + + ref, err := ec.resolveEvalRef(context.Background(), t.TempDir(), "eval_68a1f2c3") + + require.NoError(t, err) + assert.Equal(t, "eval_68a1f2c3", ref.ID) + assert.False(t, ref.Declared()) +} + +// Naming nothing where several evals are declared is ambiguous, and the +// configuration's own complaint is the useful one. +func TestResolveEvalRefReportsAmbiguity(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: first + dataset: golden + evaluators: + - evaluator: builtin.relevance + - name: second + dataset: golden + evaluators: + - evaluator: builtin.coherence +`) + ec := &evalContext{} + + _, err := ec.resolveEvalRef(context.Background(), dir, "") + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go new file mode 100644 index 00000000000..d0d0b4a5853 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" +) + +// `delete` refuses a name carried by several evals and lists the ids, so the +// order those ids come back in is part of a message a reader acts on. The +// service does not promise an order, and this used to hand back whatever the +// listing happened to contain while the comment above it claimed newest-first. +func TestEvalIDsNamedSortsNewestFirst(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_oldest", Name: "shared", CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "eval_newest", Name: "shared", CreatedAt: "2026-08-01T00:00:00Z"}, + {ID: "eval_middle", Name: "shared", CreatedAt: "2026-04-01T00:00:00Z"}, + {ID: "eval_other", Name: "different", CreatedAt: "2026-09-01T00:00:00Z"}, + }} + + got := idsNamedIn(list, "shared") + + assert.Equal(t, []string{"eval_newest", "eval_middle", "eval_oldest"}, got) +} + +// The service spells created_at as epoch seconds on some routes and RFC3339 on +// others, so both have to order the same way. +func TestEvalIDsNamedSortsAcrossTimestampShapes(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_old", Name: "shared", CreatedAt: float64(1767225600)}, // 2026-01-01 + {ID: "eval_new", Name: "shared", CreatedAt: "2026-08-01T00:00:00Z"}, + }} + + assert.Equal(t, []string{"eval_new", "eval_old"}, idsNamedIn(list, "shared")) +} + +// An eval the service described without a usable timestamp must not win by +// accident; it sorts last and the ones that can be ordered still are. +func TestEvalIDsNamedPutsUndatedLast(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_undated", Name: "shared"}, + {ID: "eval_dated", Name: "shared", CreatedAt: "2026-01-01T00:00:00Z"}, + }} + + assert.Equal(t, []string{"eval_dated", "eval_undated"}, idsNamedIn(list, "shared")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go new file mode 100644 index 00000000000..ac0222d9bdb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -0,0 +1,515 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +func newEvaluatorCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "evaluator", + Short: "Manage custom evaluators.", + } + cmd.AddCommand( + newEvaluatorCreateCommand(), + newEvaluatorUpdateCommand(), + newEvaluatorListCommand(), + newEvaluatorShowCommand(), + newEvaluatorDeleteCommand(), + newEvaluatorVersionsCommand(), + ) + return cmd +} + +// newEvaluatorCreateCommand builds `evaluator create `, which registers +// an evaluator that does not exist yet. +func newEvaluatorCreateCommand() *cobra.Command { + return newEvaluatorWriteCommand("create", "Register an evaluator, publishing its first version.") +} + +// newEvaluatorUpdateCommand builds `evaluator update `, which publishes a +// further version of one that does. +func newEvaluatorUpdateCommand() *cobra.Command { + return newEvaluatorWriteCommand("update", "Publish a new version of an evaluator.") +} + +// newEvaluatorWriteCommand builds create and update, which send the same +// request and differ only in which starting state they accept. The service has +// one route for both and assigns the version either way, so the existence check +// is ours: without it, `create` on a name already in use would silently publish +// a further version of someone else's evaluator. +func newEvaluatorWriteCommand(verb, short string) *cobra.Command { + var ( + fromFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: verb + " ", + Short: short, + Long: short + "\n\n" + + "An evaluator is a rubric: a JSON file of weighted scoring dimensions.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } + if fromFile == "" { + return requireFlag("from-file") + } + + raw, err := project.ReadFileNoBOM(fromFile) + if err != nil { + return messages.ReadingEvaluator(fromFile, err) + } + + body, err := normalizeRubricBody(name, raw) + if err != nil { + return messages.EvaluatorProblem(fromFile, err) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Asked of the direct read, not the version listing. The listing + // lags a publish by up to a second and a half, so an update + // issued straight after a create would be told the evaluator it + // just made does not exist. + existing, readErr := ec.evalClient.GetEvaluatorRaw( + ctx, name, "", ProjectEndpointAPIVersion, + ) + if readErr != nil && !eval_api.IsNotFound(readErr) { + return messages.CheckingEvaluatorExists(name, readErr) + } + // A non-404 already returned above, so reaching here means the read + // either found the evaluator or the service said it is unknown. + if err := checkAssetExistence(verb, "evaluator", name, readErr == nil, true); err != nil { + return err + } + + // What that read saw is what keeps the publish from being + // answered with the same version and replacing it. + if readErr != nil { + existing = nil + } + + created, err := ec.evalClient.CreateEvaluatorVersion( + ctx, name, body, existing, ProjectEndpointAPIVersion, + ) + if err != nil { + return messages.RegisteringEvaluator(name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprint(cmd.OutOrStdout(), + messages.EvaluatorRegistered(created.Name, created.Version)) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to the evaluator JSON file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// checkAssetExistence enforces the one difference between create and update. +// +// absenceCertain separates "the service says this name is unknown" from "nothing +// came back", and only the former refuses an update. A caller reading an +// eventually consistent version listing cannot prove absence, so an update +// issued moments after a create would otherwise be refused for a dataset that +// plainly exists -- and sent to `create`, which fails in turn once the listing +// catches up. Callers that read a point endpoint can prove it and pass true. +func checkAssetExistence(verb, kind, name string, exists, absenceCertain bool) error { + switch { + case verb == "create" && exists: + return messages.AssetAlreadyExists(kind, name) + case verb == "update" && !exists && absenceCertain: + return messages.AssetDoesNotExist(kind, name) + } + return nil +} + +// rubricDefinitionType is the discriminator the service uses to deserialize a +// rubric definition. +const rubricDefinitionType = "rubric" + +// ensureDefinitionType adds the type discriminator when a definition omits it. +// +// Without it the service cannot tell which definition kind it is holding and +// rejects the whole request with "The request field is required", which points +// at the wrong field entirely. Generated rubrics carry the type; hand-authored +// ones written to the shape the spec documents — a bare list of weighted +// dimensions — do not. +func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { + var doc map[string]json.RawMessage + if err := json.Unmarshal(definition, &doc); err != nil { + return nil, messages.DefinitionNotJSONObject(err) + } + if _, ok := doc["type"]; ok { + return definition, nil + } + doc["type"] = json.RawMessage(fmt.Sprintf("%q", rubricDefinitionType)) + return json.Marshal(doc) +} + +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. +func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, messages.NotValidJSON(err) + } + + if definition, hasDefinition := probe["definition"]; hasDefinition { + // Already a full document; make sure the name matches the argument. + typed, err := ensureDefinitionType(definition) + if err != nil { + return nil, err + } + probe["definition"] = typed + probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) + out, err := json.Marshal(probe) + if err != nil { + return nil, err + } + return out, nil + } + + if _, hasDimensions := probe["dimensions"]; !hasDimensions { + return nil, messages.RubricMissingDimensions() + } + + typed, err := ensureDefinitionType(raw) + if err != nil { + return nil, err + } + doc := map[string]any{ + "name": name, + "definition": typed, + } + out, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return out, nil +} + +func newEvaluatorListCommand() *cobra.Command { + var ( + builtin bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evaluators, or the built-in ones.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // The service filters by type, and asking for nothing returns only + // the project's own evaluators. + filter := "" + if builtin { + filter = eval_api.EvaluatorTypeBuiltin + } + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + return messages.ListingEvaluators(err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().BoolVar(&builtin, "builtin", false, + "List the built-in evaluators instead of the project's own.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newEvaluatorVersionsCommand groups the version listing, so that `list` means +// the same thing for evaluators as it does for datasets: the assets, not their +// history. +func newEvaluatorVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one evaluator.", + } + cmd.AddCommand(newEvaluatorVersionsListCommand()) + return cmd +} + +func newEvaluatorVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of an evaluator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + // A name nobody published is the ordinary way to get here, and + // it does not need the whole 404 body to explain it. + if eval_api.IsNotFound(err) { + return messages.EvaluatorNotFound(name) + } + return messages.ListingEvaluatorVersions(name, err) + } + return renderEvaluatorVersions(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + if len(list.Value) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvaluators()) + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{e.Name, e.Version, e.Type()}) + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) +} + +// evaluatorPassThreshold renders the rubric's pass mark for a table cell, +// empty when the evaluator does not carry one. +// +// Two decimals because the scale is normalized 0.0-1.0, where the difference +// between 0.7 and 0.75 is a real change in what passes. +func evaluatorPassThreshold(e *eval_api.EvaluatorSummary) string { + if e == nil || e.Definition == nil || e.Definition.PassThreshold == nil { + return "" + } + return strconv.FormatFloat(*e.Definition.PassThreshold, 'f', 2, 64) +} + +// renderEvaluatorVersions lists one evaluator's history. +// +// Name and type are the same on every row here, so they say nothing. What the +// scenario reads a version list for is how the rubric changed, which is the +// date, the pass mark and the description the author left. +func renderEvaluatorVersions(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + if len(list.Value) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvaluators()) + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{ + e.Version, + timestampString(e.CreatedAt), + evaluatorPassThreshold(&e), + e.Description, + }) + } + return emitTable(cmd.OutOrStdout(), + []string{"VERSION", "CREATED AT", "PASS THRESHOLD", "DESCRIPTION"}, rows) +} + +func newEvaluatorShowCommand() *cobra.Command { + var ( + version string + outFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an evaluator definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvaluatorNotFound(name) + } + return messages.ReadingEvaluator(name, err) + } + + // --output-file writes the service's document verbatim, because + // reconciliation points here to adopt a remote change over the local + // definition: anything this view dropped would be lost on adoption. + if outFile != "" { + body := raw + var pretty any + if err := json.Unmarshal(raw, &pretty); err == nil { + if indented, err := json.MarshalIndent(pretty, "", " "); err == nil { + body = append(indented, '\n') + } + } + if err := writeFileAtomic(outFile, body); err != nil { + return err + } + if !isJSON(cmd) { + fmt.Fprint(cmd.OutOrStdout(), messages.WroteArtifact(outFile)) + } + return nil + } + + // -o json answers with the service's document untouched, because a + // caller asking for JSON wants the evaluator, not this view of it. + if isJSON(cmd) { + var pretty any + if err := json.Unmarshal(raw, &pretty); err != nil { + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return emitJSON(cmd.OutOrStdout(), pretty) + } + + var summary eval_api.EvaluatorSummary + if err := json.Unmarshal(raw, &summary); err != nil { + // An evaluator shaped in a way this view cannot read is still + // worth showing; falling back beats refusing to print it. + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return ec.renderEvaluator(ctx, cmd.OutOrStdout(), &summary) + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&outFile, "output-file", "", + "Write the evaluator document to this path instead of stdout.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// renderEvaluator prints the detail view for one evaluator, closing with its +// portal link. +// +// The columns an evaluator is identified by, then what it grades and where it +// can run. The full definition is in `-o json`; a schema printed here would +// bury the four lines a reader came for. +func (ec *evalContext) renderEvaluator( + ctx context.Context, + out io.Writer, + e *eval_api.EvaluatorSummary, +) error { + if err := emitDetail(out, []field{ + {"Name", e.Name}, + {"Version", e.Version}, + {"Type", e.Type()}, + {"Pass Threshold", evaluatorPassThreshold(e)}, + {"Description", e.Description}, + {"Categories", strings.Join(e.Categories, ", ")}, + {"Evaluation Levels", strings.Join(e.SupportedEvaluationLevels, ", ")}, + }); err != nil { + return err + } + if prefix := ec.portalPrefix(ctx); prefix != nil && e.Name != "" { + writePortalLink(out, prefix.EvaluatorURL(e.Name, e.Version)) + } + return nil +} + +func newEvaluatorDeleteCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an evaluator version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteEvaluatorVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + if eval_api.IsNotFound(err) { + return messages.EvaluatorVersionNotFound(name, version) + } + return messages.DeletingEvaluatorVersion(name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.EvaluatorDeleted(name, version)) + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go new file mode 100644 index 00000000000..9dd8c5059a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `--evaluator a,b` used to be accepted whole, writing one evaluator literally +// named "a,b" into the config. init exited 0 and the failure surfaced two +// commands later at create, naming a value passed to a different command. +func TestEvaluatorFlagSplitsOnCommas(t *testing.T) { + cmd := newInitCommand() + require.NoError(t, cmd.Flags().Parse([]string{ + "--evaluator", "builtin.task_adherence,builtin.relevance", + })) + + got, err := cmd.Flags().GetStringSlice("evaluator") + require.NoError(t, err) + assert.Equal(t, []string{"builtin.task_adherence", "builtin.relevance"}, got, + "a comma separates references; it is never part of an evaluator name") +} + +// The sibling repeatable flag already split on commas, and two flags documented +// the same way behaving differently is visible only to someone who knows pflag. +func TestRepeatableFlagsAgreeOnCommas(t *testing.T) { + typeOf := func(cmd *pflag.FlagSet, name string) string { + f := cmd.Lookup(name) + require.NotNilf(t, f, "%s is not registered", name) + return f.Value.Type() + } + + assert.Equal(t, "stringSlice", typeOf(newInitCommand().Flags(), "evaluator")) + assert.Equal(t, typeOf(newGenerateCommand().Flags(), "from"), + typeOf(newInitCommand().Flags(), "evaluator"), + "both are repeatable reference lists, so they must split alike") +} + +// Repeating the flag still works, because a comma list is an addition rather +// than a replacement. +func TestEvaluatorFlagStillRepeats(t *testing.T) { + cmd := newInitCommand() + require.NoError(t, cmd.Flags().Parse([]string{ + "--evaluator", "builtin.task_adherence", "--evaluator", "builtin.relevance", + })) + + got, err := cmd.Flags().GetStringSlice("evaluator") + require.NoError(t, err) + assert.Equal(t, []string{"builtin.task_adherence", "builtin.relevance"}, got) +} + +// A stray comma leaves an empty reference, which would otherwise be written to +// the config and looked up as "". +func TestEvaluatorRefsRejectWhatCannotNameAnEvaluator(t *testing.T) { + require.NoError(t, validateEvaluatorRefs([]string{"builtin.relevance", "my-rubric"})) + + err := validateEvaluatorRefs([]string{"builtin.relevance", ""}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--evaluator") + + err = validateEvaluatorRefs([]string{"two words"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "two words") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go new file mode 100644 index 00000000000..e7ad1feab01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service needs a type discriminator to deserialize a definition. Without +// it the whole request is rejected with "The request field is required", which +// names the wrong field, so a hand-authored rubric failed to upload. +func TestNormalizeRubricBodyAddsDefinitionType(t *testing.T) { + raw := []byte(`{"dimensions":[{"id":"accuracy","description":"Correct.","weight":5}]}`) + + body, err := normalizeRubricBody("support-quality", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "support-quality", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) + require.Len(t, doc.Definition.Dimensions, 1) + require.Equal(t, 5, doc.Definition.Dimensions[0].Weight) +} + +// A definition that already declares its type keeps it, so a generated rubric +// round-trips unchanged. +func TestNormalizeRubricBodyKeepsExistingType(t *testing.T) { + raw := []byte(`{"type":"custom_kind","dimensions":[{"id":"a","weight":1}]}`) + + body, err := normalizeRubricBody("x", raw) + require.NoError(t, err) + + var doc struct { + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "custom_kind", doc.Definition.Type) +} + +// A full document is normalized the same way, and the name follows the flag. +func TestNormalizeRubricBodyHandlesFullDocument(t *testing.T) { + raw := []byte(`{"name":"stale","definition":{"dimensions":[{"id":"a","weight":1}]}}`) + + body, err := normalizeRubricBody("actual-name", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "actual-name", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) +} + +func TestNormalizeRubricBodyRejectsNonRubric(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"something":1}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "dimensions") + + _, err = normalizeRubricBody("x", []byte(`not json`)) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go new file mode 100644 index 00000000000..cecbf0a7129 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Evaluator versions are the unit an eval binds to, and the service assigns +// them. This proves the extension never hands back a version it has quietly +// overwritten. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorUpdateAlwaysPublishesANewVersion covers the shape of a +// first authoring session: create a rubric, look at it, change one weight, +// update. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over it rather than adding one. +// Nothing observable marks the end of that race — the version listing lags a +// publish as well, answering 404 immediately after a create — so the guard +// is the document the caller already read: it says which version exists and +// when it was written. +// +// Without it, `evaluator update` run straight after `evaluator create` reports +// success, leaves a single version holding the second rubric, and every eval +// bound to the first scores against a rubric nobody chose. +func TestLiveEvaluatorUpdateAlwaysPublishesANewVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-version-%d", time.Now().UnixNano()) + + rubric := func(weight int) json.RawMessage { + body, err := normalizeRubricBody(name, []byte(fmt.Sprintf( + `{"dimensions":[{"id":"tone","weight":%d,"description":"polite"}]}`, weight))) + require.NoError(t, err) + return body + } + + first, err := client.CreateEvaluatorVersion(ctx, name, rubric(1), nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, first.Version) + t.Cleanup(func() { + for _, v := range []string{first.Version, "1", "2"} { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, v, ProjectEndpointAPIVersion) + } + }) + + // Deliberately immediate, and passing what the caller holds rather than + // re-reading: this is the window the guard exists for, and a test that + // waited first would pass with the guard removed. + previous, err := json.Marshal(first) + require.NoError(t, err) + + started := time.Now() + second, err := client.CreateEvaluatorVersion( + ctx, name, rubric(2), previous, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEqual(t, first.Version, second.Version, + "an update issued inside the race must still publish a new version") + t.Logf("the second version was assigned after %s", time.Since(started).Round(time.Millisecond)) + + // The new version holds the new rubric, and both versions are readable. + // The earlier one is not asserted on: if the service does collide, the + // attempt that collided has already written the new definition over it, + // and no amount of care on this side can undo that. + require.Equal(t, 2, liveRubricWeight(t, client, name, second.Version)) + require.NotZero(t, liveRubricWeight(t, client, name, first.Version), + "version %s must remain readable", first.Version) +} + +// liveRubricWeight reads back the one weight the fixture rubric carries. +// +// Read as JSON rather than matched as a substring: the service reformats what +// it stores, so `"weight":1` goes in and `"weight": 1` comes back, and a +// substring assertion would fail for a reason that has nothing to do with what +// is being tested. +func liveRubricWeight( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) int { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err) + + var doc struct { + Definition struct { + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Len(t, doc.Definition.Dimensions, 1) + return doc.Definition.Dimensions[0].Weight +} + +// TestLiveFirstPublishReturnsVersionOne is the other half: the guard must not +// change what a first publish answers. +func TestLiveFirstPublishReturnsVersionOne(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-firstpub-%d", time.Now().UnixNano()) + body, err := normalizeRubricBody(name, []byte( + `{"dimensions":[{"id":"tone","weight":1,"description":"polite"}]}`)) + require.NoError(t, err) + + created, err := client.CreateEvaluatorVersion(ctx, name, body, nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + require.Equal(t, "1", created.Version, + "a name the project has never seen must publish as version 1") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go new file mode 100644 index 00000000000..e17392fd75b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "math" + "os" + "strconv" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Gating is opt-in. A completed run with failing samples exits 0 without +// --fail-on: failing samples are the expected output of a working evaluation, +// not a tool error, and `run start` is used constantly in the inner loop. A +// default that returned non-zero on any failure would break a build the first +// time a noisy grader disagreed. +// +// The separate exit code matters more than the flag. It lets a pipeline tell +// "the evaluation regressed" from "the evaluation could not run", which are +// different failures with different owners. + +// exitCodeGateBreached is returned when a run completed but missed its +// threshold. +const exitCodeGateBreached = 2 + +// gate is a parsed --fail-on threshold. +type gate struct { + set bool + anyFailure bool + passRate float64 +} + +// parseGate reads the --fail-on value. An empty value means no gating. +func parseGate(spec string) (gate, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return gate{}, nil + } + if spec == "any-failure" { + return gate{set: true, anyFailure: true}, nil + } + + rate, ok := strings.CutPrefix(spec, "pass-rate=") + if !ok { + return gate{}, messages.FailOnInvalid(spec) + } + value, err := strconv.ParseFloat(rate, 64) + if err != nil { + return gate{}, messages.FailOnRateNotNumber(rate) + } + // NaN parses, then passes both range checks, and then loses every + // comparison it is put in -- so a pipeline that asked to be gated would + // never be, and nothing would say so. + if math.IsNaN(value) { + return gate{}, messages.FailOnRateNotNumber(rate) + } + if value < 0 || value > 1 { + return gate{}, messages.FailOnRateOutOfRange(value) + } + return gate{set: true, passRate: value}, nil +} + +// scoredPassRate is the one definition of a run's pass rate: the share of the +// rows an evaluator actually scored. +// +// Errored and skipped rows are outside the denominator because nothing graded +// them, and an infrastructure failure is not a quality signal. This is what the +// portal reports and what `--fail-on pass-rate` compares against, so the two +// figures a reader sees two lines apart cannot disagree. +// +// ok is false when nothing was scored at all: a rate over no rows is not zero, +// it is absent, and the caller has to say so rather than print it. +// +// The consequence is worth stating. A run where almost everything errored can +// now report a high rate off the few rows that survived, so the count that did +// not score is printed beside it. +func scoredPassRate(counts *eval_api.EvalRunResultCounts) (rate float64, scored int, ok bool) { + if counts == nil { + return 0, 0, false + } + scored = counts.Passed + counts.Failed + if scored <= 0 { + return 0, 0, false + } + return float64(counts.Passed) / float64(scored), scored, true +} + +// breach reports why the run missed the threshold, or empty when it met it. +// +// A run that scored nothing at all breaches every threshold rather than +// dividing by zero — "no rows passed" is the honest reading of an empty result. +func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { + if !g.set { + return "" + } + if counts == nil { + return messages.GateNoResultCounts() + } + // Checked before any-failure as well as before the rate: a run that graded + // nothing has not passed, and reading zero unpassed rows as success let an + // empty run clear the gate that exists to catch exactly that. + if counts.Total == 0 { + return messages.GateNoRowsScored() + } + if g.anyFailure { + // Deliberately stricter than the rate: this counts a row nothing could + // grade against the run, because "everything passed" is not true of a + // run that failed to grade half of what it was given. + unpassed := counts.Total - counts.Passed + if unpassed > 0 { + return messages.GateSamplesDidNotPass(unpassed, counts.Total) + } + return "" + } + actual, _, ok := scoredPassRate(counts) + if !ok { + return messages.GateNoRowsScored() + } + if actual < g.passRate { + return messages.GatePassRateBelow(actual, g.passRate) + } + return "" +} + +// gateBreachMessage is what a breached gate prints, kept separate from the +// exit so the wording can be tested: it is the block the spec's CI scenario +// shows, and a pipeline's logs are where it is read. +func gateBreachMessage(reason string) string { + return messages.GateBreached(reason) +} + +// applyGate ends the process with exit code 2 when the run missed its +// threshold. +// +// It exits here rather than returning an error because the extension SDK's +// Run collapses every error to exit 1, and the whole point of the flag is a +// code a pipeline can tell apart from an operational failure. +func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { + if run == nil { + return + } + // Rows nothing could grade are outside the rate, so a run that errored on + // most of what it was given can clear a threshold on the few that survived. + // That is the cost of measuring quality over scored rows only, and the gate + // is where it has to be said: this is the line a pipeline log keeps. + if g.set && !g.anyFailure { + if c := run.ResultCounts; c != nil { + if _, scored, ok := scoredPassRate(c); ok && c.Total > scored { + fmt.Fprint(os.Stderr, + messages.Warning(messages.GateSawUnscoredRows(c.Total-scored, c.Total))) + } + } + } + reason := g.breach(run.ResultCounts) + if reason == "" { + return + } + fmt.Fprint(os.Stderr, gateBreachMessage(reason)) + os.Exit(exitCodeGateBreached) +} + +func addFailOnFlag(cmd *cobra.Command, target *string) { + // States the observed code rather than the one this process exits with: azd + // collapses an extension's exit code, and a pipeline author who reads 2 here + // writes a condition that never fires. + cmd.Flags().StringVar(target, "fail-on", "", + "Fail when the run misses this threshold: any-failure, or pass-rate=<0..1>. "+ + "pass-rate is measured over the rows that were scored, so rows nothing "+ + "could grade are outside it; any-failure counts them against the run. "+ + "Exits 1.") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go new file mode 100644 index 00000000000..8b137b96a0b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "strings" + "testing" + "time" + + "azureaieval/internal/messages" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run that outlives the wait leaves --fail-on with nothing to judge. Exiting +// 0 there tells a pipeline the gate passed, which is the silent drop --no-wait +// is refused for, reached by running long instead. The message has to say the +// gate did not run, not merely that the wait stopped. +func TestGateOutlivedTheWaitSaysTheGateNeverRan(t *testing.T) { + err := messages.GateOutlivedTheWait("run_abc", 2*time.Hour) + + require.Error(t, err) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), "2h0m0s", "the message has to say how long it waited") + assert.Contains(t, err.Error(), "--fail-on", + "a reader has to know which flag went unanswered") + assert.Contains(t, err.Error(), "run show", + "and how to get the verdict they asked for") +} + +// The check above only proves the message is right, not that anything calls it. +// Driving the branch itself needs a run that outlives a two-hour const budget +// and a signed-in client, so this reads the source instead. +// +// Worth the ugliness here: the failure mode is a gate that passes silently, so +// a regression looks exactly like success and no other test would notice. Same +// reasoning as the linker-path check in internal/version. +func TestWaitBudgetBranchStillConsultsTheGate(t *testing.T) { + body, err := os.ReadFile("run.go") + require.NoError(t, err) + + src := string(body) + start := strings.Index(src, "errors.Is(err, errWaitBudgetSpent)") + require.NotEqual(t, -1, start, "the wait-budget branch has moved or gone") + + branch := src[start:] + if end := strings.Index(branch, "\n\t\t\tif err != nil {"); end != -1 { + branch = branch[:end] + } + + assert.Contains(t, branch, "threshold.set", + "the branch has to ask whether a gate was set before reporting success") + assert.Contains(t, branch, "GateOutlivedTheWait", + "and refuse rather than exit 0 when one was") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go new file mode 100644 index 00000000000..d6757667900 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's CI scenario prints this block verbatim, and a pipeline's log is +// where it is read. Both lines are asserted whole: the marker is what tells a +// reader this is a gate rather than a crash, and the ERROR: line is what the +// azd style guide reserves for a terminal failure. +func TestGateBreachMessageMatchesTheScenario(t *testing.T) { + msg := gateBreachMessage("pass rate 76.0% is below the required 80.0%") + + assert.Equal(t, + "(x) Failed: Evaluation gate: pass rate 76.0% is below the required 80.0%\n\n"+ + "ERROR: evaluation quality gate not met.\n", + msg) +} + +// Exit 2 is the whole point of --fail-on: a pipeline has to tell "the +// evaluation regressed" apart from "the tool could not run", which is exit 1. +func TestGateBreachUsesItsOwnExitCode(t *testing.T) { + assert.Equal(t, 2, exitCodeGateBreached, + "the spec's exit table gives 2 to a breached threshold") +} + +// The spec's exit table: a completed run is 0 whatever it scored, and a run +// that errored rather than completed is an operational failure. +func TestRunCompletedSeparatesRegressionFromFailureToRun(t *testing.T) { + for _, status := range []string{"completed", "Completed", ""} { + assert.NoErrorf(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r", Status: status}), + "%q is a run that produced results, so its score decides the outcome", status) + } + + for _, status := range []string{"failed", "errored", "canceled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: status}) + require.Errorf(t, err, "%q never produced results, so it did not regress", status) + assert.Contains(t, err.Error(), status) + assert.Contains(t, err.Error(), "r1") + } +} + +// pass-rate is passed/(passed+failed): the share of the rows something actually +// graded. Errored and skipped rows are outside it, because an infrastructure +// failure is not a quality signal, and this is the figure the portal reports. +// +// The cost is real and deliberate: a run with two passes and thirteen errors +// scores a perfect rate. `any-failure` is the gate that still counts those, and +// a pass-rate gate warns when it judged only part of a run. +func TestPassRateIsMeasuredOverTheRowsThatWereScored(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + // 2 passed, 1 failed, 1 errored: 2 of 3 scored is 66.7%, below 80%. + breach := g.breach(&eval_api.EvalRunResultCounts{Total: 4, Passed: 2, Failed: 1, Errored: 1}) + require.NotEmpty(t, breach, "a scored row that failed still counts") + assert.Contains(t, breach, "66.7%") + + // The same run without the failure: everything scored, passed, so the + // errored row does not drag a quality number down on its own. + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "nothing graded the errored row, so it is not evidence of a regression") + + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3})) +} + +// any-failure is the same concern as the rate above asked as a yes or no, and +// it was the untested half: the gate counts everything that is not a pass, so +// replacing that with the Failed count alone let a run whose rows errored +// report success, and no test noticed. +func TestAnyFailureCountsErroredAndSkippedAsUnpassed(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "a row that errored did not pass") + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Skipped: 1}), + "a row that was skipped did not pass either") + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Failed: 1})) + + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3}), + "every row passed, so there is nothing to report") +} + +// A run that scored nothing breaches every threshold rather than dividing by +// zero. "No rows passed" is the honest reading of an empty result. +func TestEmptyRunBreachesEveryThreshold(t *testing.T) { + g, err := parseGate("pass-rate=0.1") + require.NoError(t, err) + + breach := g.breach(&eval_api.EvalRunResultCounts{}) + + assert.NotEmpty(t, breach) + assert.NotContains(t, breach, "NaN", "dividing by zero must not reach the message") +} + +// --fail-on belongs to the commands that wait for a terminal state, so a +// pipeline that started a run asynchronously can still gate where it reattaches. +func TestFailOnSitsOnTheWaitingCommands(t *testing.T) { + for _, path := range []string{"run start", "run show"} { + assert.NotNilf(t, find(t, path).Flags().Lookup("fail-on"), + "%s waits for a terminal state, so it can gate on one", path) + } + + usage := find(t, "run start").Flags().Lookup("fail-on").Usage + for _, form := range []string{"any-failure", "pass-rate"} { + assert.Containsf(t, usage, form, "--fail-on accepts %q, so its help has to say so", form) + } + assert.Contains(t, strings.ToLower(usage), "1", + "the help has to name the exit code a caller observes, which is the only reason to use the flag") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go new file mode 100644 index 00000000000..6a0d5f208f3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `--no-wait --fail-on ...` reads as "start it and tell me if it regressed". +// It cannot be: --no-wait returns before there is a result, so the gate was +// dropped and the command exited 0 however the run turned out. A pipeline +// written that way believes it is gated and is not, which is worse than not +// gating at all. +// +// Refused up front, before any network work. +func TestFailOnWithNoWaitIsRefused(t *testing.T) { + for _, gate := range []string{"any-failure", "pass-rate=0.8"} { + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"run", "start", "--no-wait", "--fail-on", gate}) + + err := root.ExecuteContext(context.Background()) + + require.Errorf(t, err, "--no-wait with --fail-on %s must not be accepted", gate) + assert.Contains(t, err.Error(), "--fail-on") + assert.Contains(t, err.Error(), "--no-wait") + assert.Containsf(t, err.Error(), "run show", + "the refusal has to name the way to gate a run started with --no-wait") + } +} + +// The gate on its own still parses and still reaches the run, so the refusal +// above is about the combination and not about --fail-on. +func TestFailOnAloneIsStillAccepted(t *testing.T) { + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"run", "start", "--fail-on", "pass-rate=0.8"}) + + err := root.ExecuteContext(context.Background()) + + if err != nil { + assert.NotContains(t, err.Error(), "--no-wait", + "a gate without --no-wait must not be refused for needing the wait") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go new file mode 100644 index 00000000000..37d6e56afef --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A gate is the whole reason a pipeline runs this command, so the ways it can +// silently stop gating matter more than the ways it can fire. + +// NaN parses, clears both range checks, and then loses every comparison it is +// put in. A pipeline written this way believes it is gated and is not. +func TestFailOnRejectsAThresholdThatCanNeverFire(t *testing.T) { + for _, spec := range []string{"pass-rate=NaN", "pass-rate=nan", "pass-rate=-nan"} { + _, err := parseGate(spec) + require.Errorf(t, err, "%s would disable the gate while looking like one", spec) + } + + // The range check already covers infinities; this pins that it still does. + for _, spec := range []string{"pass-rate=Inf", "pass-rate=-Inf", "pass-rate=1.5", "pass-rate=-0.1"} { + _, err := parseGate(spec) + require.Errorf(t, err, "%s is not a pass rate", spec) + } + + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + assert.True(t, g.set) + assert.InDelta(t, 0.8, g.passRate, 1e-9) +} + +// A run that graded nothing has not passed. The pass-rate gate always said so; +// any-failure computed Total-Passed, which is zero for an empty run, and let it +// through -- the one shape a gate exists to catch. +func TestEveryGateBreachesOnARunThatScoredNothing(t *testing.T) { + empty := &eval_api.EvalRunResultCounts{Total: 0} + + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + assert.NotEmpty(t, anyFailure.breach(empty), + "an empty run must not clear an any-failure gate") + + rate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + assert.NotEmpty(t, rate.breach(empty)) + + // A gate that was never asked for stays silent whatever the counts. + assert.Empty(t, gate{}.breach(empty)) +} + +// The ordinary cases still behave, so the empty-run guard did not swallow them. +func TestGatesStillJudgeRunsThatScoredSomething(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + + assert.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3}), + "every row passed, so there is nothing to report") + assert.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2}), + "one row did not pass") + + rate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + // Spelled out with Failed rather than left to total, because the rate is + // measured over what was scored: passed plus failed. + assert.Empty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 9, Failed: 1})) + assert.NotEmpty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 3})) + + // Counts the service never sent are not a pass. + assert.NotEmpty(t, rate.breach(nil)) +} + +// The spec puts --fail-on on the commands that wait. A run still in progress +// has partial counts, so gating it can fail a run that would have passed -- +// and silently skipping the gate would leave a pipeline believing it is +// protected when it is not. +func TestOnlyATerminalRunCanBeGated(t *testing.T) { + // Read from the polling vocabulary instead of a second copy of it. Spelling + // the list out here is what hid "error" being gateable: the test agreed with + // the bug. + for status := range terminalRunStates { + assert.Truef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), + "the poller stops on %q, so its counts are final", status) + } + assert.True(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: ""}), + "a run the service reported no status for is gated on the counts it gave") + + for _, status := range []string{"in_progress", "queued", "running"} { + assert.Falsef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), + "%q is still moving, so its counts are partial", status) + } + + assert.False(t, runIsTerminal(nil), "no run is not a finished run") + + err := messages.GateNeedsATerminalRun("evalrun_1", "in_progress") + require.Error(t, err) + assert.Contains(t, err.Error(), "--wait", "the way out has to be named") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go new file mode 100644 index 00000000000..f723bc6ebd2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestParseGate(t *testing.T) { + t.Run("empty means no gating", func(t *testing.T) { + g, err := parseGate("") + require.NoError(t, err) + require.False(t, g.set) + require.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3}), + "an unset gate must never breach") + }) + + t.Run("any-failure", func(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + require.True(t, g.anyFailure) + }) + + t.Run("pass-rate", func(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + require.InDelta(t, 0.8, g.passRate, 1e-9) + }) + + for _, bad := range []string{"passrate=0.8", "pass-rate=abc", "pass-rate=1.5", "pass-rate=-1", "sometimes"} { + t.Run("refuses "+bad, func(t *testing.T) { + _, err := parseGate(bad) + require.Error(t, err) + }) + } +} + +func TestGateBreach(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + eighty, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + t.Run("any-failure passes only when every row passed", func(t *testing.T) { + require.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 2})) + require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) + }) + + // Errored rows sit outside the rate: nothing graded them, so they are not + // evidence of a regression. `any-failure` is the gate that counts them. + t.Run("errored rows are outside the rate", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 12, Passed: 8, Failed: 2, Errored: 2} + require.Equal(t, "", eighty.breach(counts), "8 of the 10 scored passed, which meets 0.8") + + counts = &eval_api.EvalRunResultCounts{Total: 13, Passed: 7, Failed: 3, Errored: 3} + require.NotEmpty(t, eighty.breach(counts), "7 of the 10 scored is under 0.8") + + counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} + require.Empty(t, eighty.breach(counts), + "everything that was scored passed, so the errored rows do not breach it") + }) + + // The wording is pinned because the hero scenario shows it verbatim. + t.Run("reads as a percentage", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764, Failed: 236} + require.Equal(t, + "pass rate 76.4% is below the required 80.0%", + eighty.breach(counts)) + }) + + // A run that scored nothing has no defensible pass rate, and treating it as + // 100% would let a broken evaluation hold a gate open. + t.Run("a run that scored nothing breaches", func(t *testing.T) { + require.NotEmpty(t, eighty.breach(&eval_api.EvalRunResultCounts{Total: 0})) + require.NotEmpty(t, eighty.breach(nil)) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go new file mode 100644 index 00000000000..3f0c45a4263 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -0,0 +1,494 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. +// The generation job is not gateway-capped; the old limit simply gave up while +// the service was still working, forcing a second command. +var generatePollBudget = eval_api.PollerOptions{ + Interval: 5 * time.Second, + MaxAttempts: 720, // one hour +} + +// generationPlan is everything one generation job needs, after the flags, the +// generation spec, and the eval's own target have been reconciled. +type generationPlan struct { + // Name of the artifact being generated — the positional argument. + Name string + // Agent whose context seeds generation. May be empty, in which case + // generation runs from the instruction alone. + Agent string + // Model deployment the generation job runs against. + Model string + // Instruction describing what the agent does and what to test. + Instruction string + // BaseDir is the directory OutputDir resolves against. + BaseDir string + // OutputDir is where the artifact is written. + OutputDir string + // SampleSize applies to dataset generation only. + SampleSize int + // From is what --from named: which of the service's sources to send. Empty + // sends whatever the plan has to offer. + From []string + // TraceDays seeds generation from that many days of recent traces. + TraceDays int + // Kind is which artifact this plan produces, so one runner can submit both. + Kind generateKind +} + +// generateKind names the two generation resources, which share no collection. +type generateKind string + +const ( + generateKindDataset generateKind = "dataset" + generateKindEvaluator generateKind = "evaluator" +) + +// traceOptions converts the plan's trace window into the generation client's +// day count. Traces seed generation only; they are never a run's data source. +func (p generationPlan) traceOptions() *eval_api.TraceOptions { + if p.TraceDays <= 0 { + return nil + } + return &eval_api.TraceOptions{Days: p.TraceDays} +} + +// resolveInstruction returns the generation instruction, reading it from a +// file when one is named. +// +// A useful instruction describes the agent and what to test, which is often +// more than fits comfortably on a command line, so it can live in a file that +// is reviewable alongside the rest of the config. +func resolveInstruction(inline, path string) (string, error) { + if path == "" { + return inline, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", messages.ReadingInstructionFile(path, err) + } + text := strings.TrimSpace(string(raw)) + if text == "" { + return "", messages.InstructionFileEmpty(path) + } + return text, nil +} + +// declaredInstructions reads the file named by a generation entry's +// `instructions`, relative to the spec that declared it. +// +// A missing file is not an error. The path can be written before the file +// exists, so treating its absence as a failure would break the flow `init` +// scaffolds. +func declaredInstructions(named, configPath string) (string, error) { + if named == "" { + return "", nil + } + + path := named + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(configPath), filepath.FromSlash(named)) + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", messages.ReadingInstructions(named, err) + } + return strings.TrimSpace(string(raw)), nil +} + +// resolveGenerationInstruction decides what generation is seeded from. +// +// The service accepts an agent source that is meant to pull the agent's own +// instructions, but it fails for every agent, so the agent's context is read +// here instead. In precedence order: what the caller passed, the instructions +// the project already holds, then the agent's published ones. +// +// The project comes before the service because a local read cannot fail +// slowly, and because instructions that have been optimized but not yet +// deployed are the ones the author means — generating against what is still +// published would test the version they are replacing. +// +// The last step is what makes `generate` work with no authored input at all, +// which is the flow `init` sets up. +func (ec *evalContext) resolveGenerationInstruction( + ctx context.Context, + explicit, agentName string, + out io.Writer, + quiet bool, +) (string, error) { + if explicit != "" { + return explicit, nil + } + + if agentName == "" { + return "", nil + } + + local, path, err := ec.agentInstructionsFromProject(ctx, agentName) + if err != nil { + return "", err + } + if local != "" { + if !quiet { + fmt.Fprint(out, messages.SeedingFromFile(filepath.ToSlash(path))) + } + return local, nil + } + + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) + if err != nil { + // Reported without stopping, because the model can still be supplied by + // --generation-model and the caller has its own checks for what is left + // missing. Making an absent agent fatal here reads well for a typo but + // takes away the only path to "nothing supplied a model", which is the + // case the flag validation exists for. + if !quiet { + fmt.Fprint(out, messages.WarningAgentUnreadable(agentName, err)) + } + return "", nil + } + instructions := agent.Instructions() + if instructions != "" && !quiet { + fmt.Fprint(out, messages.SeedingFromAgent(agentName)) + } + return instructions, nil +} + +// agentInstructionsFromProject reads the agent's instructions out of the azd +// project, coming back empty when there is no project to read. +// +// Running outside a project is ordinary — the atomic commands work standalone +// against the data plane — so not finding one is not an error. An ambiguous +// target inside one is, because it would otherwise pick an agent at random. +func (ec *evalContext) agentInstructionsFromProject( + ctx context.Context, + agentName string, +) (instruction string, path string, err error) { + if ec.azdClient == nil { + return "", "", nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", "", nil + } + return project.AgentInstructionsFromProject(resp.GetProject(), agentName) +} + +// generateRubric submits the evaluator generation job and saves the rubric. +func (ec *evalContext) generateRubric( + ctx context.Context, + plan generationPlan, + out io.Writer, + noWait bool, + // jobID receives the submitted job's id, for the same reason as above. + jobID *string, +) (*project.ArtifactRef, error) { + fmt.Fprint(out, messages.GeneratingRubric(plan.Name)) + + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), + ) + if err := refuseUnusableSources(sources, unbuildable); err != nil { + return nil, err + } + req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) + + job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) + if err != nil { + return nil, messages.SubmittingRubricJob(err) + } + if jobID != nil { + *jobID = job.ID + } + if noWait { + reportSubmitted(out, "evaluator", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, ProjectEndpointAPIVersion, + ec.evalClient.GetEvaluatorGenerationJob) + if err != nil { + return nil, messages.RubricGeneration(err) + } + + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".json") + if err := writeRubric(path, completed.Result); err != nil { + return nil, err + } + fmt.Fprint(out, messages.WroteArtifact(path)) + + _, version := completed.ResolvedNameVersion() + return &project.ArtifactRef{ + Name: plan.Name, + Source: relativeSource(plan.BaseDir, path), + Version: version, + }, nil +} + +// refuseUnbuildableSources reports a --from the plan could not honour. +// +// Submitting anyway would run a billed job seeded from less than was asked for +// and return a plausible-looking artifact, which is the worst outcome: the +// caller has no way to tell it apart from one built the way they intended. +// refuseUnusableSources rejects a generation the service could only refuse. +// +// Unbuildable kinds each get their own reason. Beyond those, a request with no +// sources at all is refused here rather than sent: a kind can be selected +// without being asked for and without anything to build it from, which added to +// neither list, so an empty request went out and came back as a 400 wrapping +// thirty lines of JSON around one sentence. +func refuseUnusableSources(sources []eval_api.GenerationSource, kinds []string) error { + if err := refuseUnbuildableSources(kinds); err != nil { + return err + } + if len(sources) == 0 { + return messages.NothingToGenerateFrom() + } + return nil +} + +func refuseUnbuildableSources(kinds []string) error { + if len(kinds) == 0 { + return nil + } + reasons := map[string]string{ + "prompt": messages.FromPromptNeedsInstruction(), + "agent": messages.FromAgentNeedsTarget(), + "file": messages.FromFileNotASource(), + } + reasonsForKinds := make([]string, 0, len(kinds)) + for _, k := range kinds { + if reason, ok := reasons[k]; ok { + reasonsForKinds = append(reasonsForKinds, reason) + continue + } + reasonsForKinds = append(reasonsForKinds, messages.FromNotBuildable(k)) + } + return messages.UnbuildableSources(reasonsForKinds) +} + +// reportSubmitted says what was started and how to get back to it. +// +// The job id goes into the command rather than being left as a placeholder: +// --no-wait exists so the caller can walk away, and the line they walk away +// with has to be the one they can paste when they come back. The group is named +// too, because the two job types share no collection. +func reportSubmitted(out io.Writer, group, jobID string) { + fmt.Fprint(out, messages.JobSubmitted(jobID)) + fmt.Fprint(out, messages.ReattachToJob(group, jobID)) +} + +// generateDataset submits the data generation job and downloads the result. +func (ec *evalContext) generateDataset( + ctx context.Context, + plan generationPlan, + out io.Writer, + noWait bool, + // jobID receives the submitted job's id. Under --no-wait nothing is + // downloaded and there is no artifact to return, so this is the only thing + // the caller can report or reattach to. + jobID *string, +) (*project.ArtifactRef, error) { + fmt.Fprint(out, messages.GeneratingDataset(plan.Name, plan.SampleSize)) + + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), + ) + if err := refuseUnusableSources(sources, unbuildable); err != nil { + return nil, err + } + req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) + + job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, messages.SubmittingDataJob(err) + } + if jobID != nil { + *jobID = job.ID + } + if noWait { + reportSubmitted(out, "dataset", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + if err != nil && isAgentSeededGenerationFailure(err) { + // Agent-seeded generation fails server-side for every agent, while the + // same request carrying only the prompt succeeds. Failing the whole + // command would block the documented flow on a defect the user cannot + // do anything about, so retry without the agent and say so. + promptOnly := eval_api.WithoutAgentSource(sources) + if eval_api.HasPromptSource(promptOnly) { + fmt.Fprint(out, messages.WarningAgentSeedFailedRetrying(plan.Agent)) + + req = eval_api.NewDataGenerationJobRequest( + plan.Name, plan.Model, plan.SampleSize, promptOnly) + job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, messages.SubmittingDataJob(err) + } + // The retry is a second billed job, so the id the caller reports + // has to move with it. Leaving it on the abandoned first job points + // every resume and every `job show` at the wrong one. + if jobID != nil { + *jobID = job.ID + } + completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + } + } + if err != nil { + return nil, messages.DataGeneration(explainDataGenerationFailure(err, plan.Agent)) + } + + name, version := completed.ResolvedNameVersion() + if name == "" { + return nil, messages.DataJobReturnedNoDataset() + } + + // Confirm the version exists before reading it, so a missing dataset is + // reported as such rather than as a download failure. + if _, err := ec.datasetClient.GetDataset( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return nil, messages.ReadingGeneratedDataset(name, err) + } + content, err := ec.datasetClient.DownloadDatasetContent(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, messages.DownloadingGeneratedDataset(name, err) + } + + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, messages.Creating(filepath.Dir(path), err) + } + if err := os.WriteFile(path, content, 0o600); err != nil { + return nil, messages.Writing(path, err) + } + fmt.Fprint(out, messages.WroteArtifact(path)) + + // The job registered the version and this file is a copy of it, so the + // state a deploy would have left behind is recorded now. Without it the + // next `azd up` finds no fingerprint for this dataset, reads the file as + // new, and publishes a second version identical to the one just generated. + ec.recordDeployedDataset(ctx, plan.Name, path, version) + + return &project.ArtifactRef{ + Name: plan.Name, + Source: relativeSource(plan.BaseDir, path), + Version: version, + }, nil +} + +// isAgentSeededGenerationFailure recognizes the service-side failure that hits +// every agent, so it can be retried without the agent rather than surfaced. +func isAgentSeededGenerationFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "DataGenerationJobSystemError") || + strings.Contains(text, "Something went wrong during data generation") +} + +// explainDataGenerationFailure adds context to the service's opaque system +// error. +// +// Seeding generation from an agent currently fails server-side with +// DataGenerationJobSystemError for every agent, within seconds, while the same +// request without the agent source runs normally. The raw message says only +// that something went wrong and to try again, which sends users into a retry +// loop against a deterministic failure. +func explainDataGenerationFailure(err error, agentName string) error { + if err == nil || agentName == "" { + return err + } + // The poller surfaces the service's message; the code is not always in it. + text := err.Error() + if !strings.Contains(text, "DataGenerationJobSystemError") && + !strings.Contains(text, "Something went wrong during data generation") { + return err + } + return messages.AgentSeededGenerationFailing(err, agentName) +} + +// pollGeneration waits for a generation job using the raised budget. +func (ec *evalContext) pollGeneration( + ctx context.Context, + operationID, apiVersion string, + get eval_api.GetJobFunc, +) (*eval_api.GenerationJob, error) { + poller := eval_api.NewPoller(operationID, apiVersion, get) + poller.Options = generatePollBudget + return poller.Poll(ctx) +} + +// writeRubric persists the rubric so the developer can edit weights and +// descriptions and publish a new version. +// +// The definition is written through as it arrived rather than re-marshalled +// from a struct. Re-marshalling keeps only the fields the struct models, and +// dropped pass_threshold: the file then differed from the version that had just +// been published, so the next deploy republished it, silently without a +// threshold. Anything the service adds later would have been lost the same way. +func writeRubric(path string, result json.RawMessage) error { + if len(result) == 0 { + return messages.RubricJobReturnedNoResult() + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return messages.Creating(filepath.Dir(path), err) + } + + var envelope struct { + Definition json.RawMessage `json:"definition"` + } + if err := json.Unmarshal(result, &envelope); err == nil && len(envelope.Definition) > 0 { + var probe struct { + Dimensions []json.RawMessage `json:"dimensions"` + } + if json.Unmarshal(envelope.Definition, &probe) == nil && len(probe.Dimensions) > 0 { + var pretty bytes.Buffer + if err := json.Indent(&pretty, envelope.Definition, "", " "); err != nil { + return messages.Serializing(path, err) + } + return os.WriteFile(path, pretty.Bytes(), 0o600) + } + } + + // Fall back to the raw payload rather than losing the result. + return os.WriteFile(path, result, 0o600) +} + +// relativeSource expresses an artifact path relative to the deployment spec. +func relativeSource(baseDir, path string) string { + rel, err := filepath.Rel(baseDir, path) + if err != nil { + return filepath.ToSlash(path) + } + return "./" + filepath.ToSlash(rel) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go new file mode 100644 index 00000000000..a0246c5e955 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation is split per artifact because the service splits it: datasets and +// evaluators are separate long-running resources. One composite verb leaves +// partial failure undefined, cannot regenerate one artifact after the other has +// been hand-edited, and gives --no-wait nothing to reattach to. +// +// Neither command edits azure.yaml. Both add a catalog entry to azure.eval.yaml for +// what they produced, so the artifact is referenceable without a hand edit. + +// generateFlags are the settings both generate commands share. +// +// There is no generation spec file. Every setting is a flag, because the +// artifact is checked in and a regeneration usually wants different settings +// anyway; what that costs is provenance, which is Open Question 8. +type generateFlags struct { + path string + target string + instruction string + instructionFile string + model string + outputDir string + noWait bool + force bool + endpoint string +} + +func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { + cmd.Flags().StringVar(&f.path, "path", "", + "Directory holding the evaluation configuration. Defaults to the directory "+ + "`init` scaffolded, otherwise ./evals.") + cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&f.instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") + cmd.Flags().StringVar(&f.model, "generation-model", "", + "Model deployment that generates the artifact.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", "", + "Directory the generated artifact is written to.") + cmd.Flags().BoolVar(&f.noWait, "no-wait", false, + "Submit the job and return its id without polling.") + cmd.Flags().BoolVar(&f.force, "force", false, + "Overwrite an artifact file that already exists.") + cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") +} + +// resolvePlan settles every input that does not need the network. +// +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. The instruction +// file is read here rather than later so that an input the caller named and got +// wrong is reported ahead of one they simply left out. +func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declaredTarget(f.path)), + Model: f.model, + Instruction: instruction, + BaseDir: f.path, + OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), + } + if plan.Model == "" && plan.Agent == "" { + return plan, messages.GenerationModelRequired() + } + return plan, nil +} + +// prepareGeneration builds the client and settles the two inputs that need it: +// the agent's published instructions, and its deployment when the caller named +// no model of its own. Only the service can supply either. +func prepareGeneration( + cmd *cobra.Command, + f *generateFlags, + plan generationPlan, +) (*evalContext, generationPlan, error) { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) + if err != nil { + return nil, plan, err + } + + plan.Instruction, err = ec.resolveGenerationInstruction( + ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, plan, err + } + + if plan.Model == "" { + plan.Model = ec.agentDeployment(ctx, plan.Agent, cmd.OutOrStdout(), isJSON(cmd)) + } + if plan.Model == "" { + ec.Close() + return nil, plan, messages.GenerationModelRequired() + } + return ec, plan, nil +} + +// agentDeployment reads the deployment the target agent answers with. +// +// Best effort, but not silent: a misspelled --target and an agent with no +// published version both end in "pass --generation-model", which names neither. +// The warning is what tells those two apart. +func (ec *evalContext) agentDeployment( + ctx context.Context, + agentName string, + out io.Writer, + quiet bool, +) string { + if agentName == "" { + return "" + } + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) + if err != nil { + if !quiet { + fmt.Fprint(out, messages.CouldNotReadAgentForModel(agentName, err)) + } + return "" + } + return agent.Model() +} + +// declaredTarget reads the agent from the evaluation configuration, which is +// where the target is already declared, so `generate` does not need it +// repeated. Best effort: generation runs from the instruction alone when there +// is no configuration to read, which is the case in a bare directory. +func declaredTarget(evalDir string) string { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil || cfg == nil { + return "" + } + for _, eval := range cfg.Evals { + if eval.Target != nil && eval.Target.Name != "" { + return eval.Target.Name + } + } + return "" +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// refuseExistingArtifact stops a generation that would overwrite a checked-in +// file, because the job is billed and the diff is what the author reviews. +func refuseExistingArtifact(path string, force bool) error { + if force { + return nil + } + if _, err := os.Stat(path); err == nil { + return messages.ArtifactExists(filepath.ToSlash(path)) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go new file mode 100644 index 00000000000..59eee0fbbed --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -0,0 +1,395 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + "sync" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// One verb with a selector, per the spec. +// +// The two artifacts are separate long-running service resources, but a +// developer starting out wants both and should not have to know that. Omitting +// both flags generates both; passing one narrows generation to it, which is +// also how you regenerate one after the other has been hand-edited. +// +// The jobs are submitted together because neither is an input to the other. +// Their output is buffered and replayed in a fixed order rather than written as +// it arrives: two generations reporting progress into the same terminal +// interleave into nonsense. The catalog is written after both have finished, +// on this goroutine, because both entries land in the same file. + +func newGenerateCommand() *cobra.Command { + var ( + flags generateFlags + maxSamples int + from []string + traceDays int + wantDataset bool + wantEvaluator bool + datasetName string + evaluatorName string + ) + + cmd := &cobra.Command{ + Use: "generate", + Short: "Generate a dataset and a rubric evaluator, and download them.", + Long: "Generate a dataset and a rubric evaluator, and download them.\n\n" + + "Both are produced unless --dataset or --evaluator narrows it to one. " + + "Neither is an input to the other, so the jobs run together and each " + + "reports its own outcome; the command fails if either did.\n\n" + + "--from selects one or more of the sources the service generates the " + + "dataset from, and is repeatable.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + dataset, evaluator := selectedArtifacts(wantDataset, wantEvaluator) + + // Checked before any network work, so a flag that cannot apply + // costs nothing to find out about. Changed() rather than the value, + // so a zero the caller actually typed is still caught and an + // untouched default is not. + if err := refuseInapplicableFlags(cmd, dataset, evaluator); err != nil { + return err + } + if traceDays < 0 { + return messages.NegativeTraceDays(traceDays) + } + if flags.noWait && cmd.Flags().Changed("output-dir") { + return messages.OutputDirNeedsTheWait() + } + + if dataset { + for _, src := range from { + if err := project.ValidateGenerateSource(src); err != nil { + return err + } + } + if err := project.ValidateSampleSize(maxSamples); err != nil { + return err + } + } + + // Settled before anything reads or writes the configuration, so the + // catalog entry lands next to the eval `init` scaffolded rather than + // in a second configuration under ./evals that nothing else reads. + resolvedPath, err := resolveEvalDir(cmd.Context(), flags.path) + if err != nil { + return err + } + flags.path = resolvedPath + + target := firstNonEmpty(flags.target, declaredTarget(flags.path)) + plans, err := buildGeneratePlans(generateRequest{ + flags: &flags, + target: target, + dataset: dataset, + evaluator: evaluator, + datasetName: datasetName, + evaluatorName: evaluatorName, + maxSamples: maxSamples, + from: from, + traceDays: traceDays, + }) + if err != nil { + return err + } + + ec, resolved, err := prepareGeneration(cmd, &flags, plans[0]) + if err != nil { + return err + } + defer ec.Close() + + // prepareGeneration settles the inputs only the service can supply. + // They are the same for both artifacts, so they are read once. + for i := range plans { + plans[i].Instruction = resolved.Instruction + plans[i].Model = resolved.Model + } + if dataset && len(plans[0].From) == 0 { + plans[0].From = defaultGenerationSource( + ec.getEnvValue(cmd.Context(), appInsightsEnvKey), + ) + } + + return ec.runGenerations(cmd, plans, flags) + }, + } + + cmd.Flags().BoolVar(&wantDataset, "dataset", false, + "Generate only the dataset. Omit both flags to generate both.") + cmd.Flags().BoolVar(&wantEvaluator, "evaluator", false, + "Generate only the evaluator. Omit both flags to generate both.") + cmd.Flags().StringVar(&datasetName, "dataset-name", "", + "Name for the generated dataset. Defaults to -dataset.") + cmd.Flags().StringVar(&evaluatorName, "evaluator-name", "", + "Name for the generated evaluator. Defaults to -evaluator.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d. Dataset only.", + project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where the dataset's rows come from: %s. Repeatable, and the "+ + "service accepts more than one. Defaults to %s when the project has "+ + "Application Insights connected, otherwise %s. Dataset only.", + strings.Join(project.GenerateSources, ", "), + project.GenerateFromTraces, project.GenerateFromAgent)) + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed the evaluator's rubric. 0 disables.") + addGenerateFlags(cmd, &flags) + return cmd +} + +// selectedArtifacts reads the pair of narrowing flags. Neither set means both, +// which is the zero-to-first-eval path the composite exists for. +func selectedArtifacts(dataset, evaluator bool) (bool, bool) { + if !dataset && !evaluator { + return true, true + } + return dataset, evaluator +} + +type generateRequest struct { + flags *generateFlags + target string + dataset bool + evaluator bool + datasetName string + evaluatorName string + maxSamples int + from []string + traceDays int +} + +// artifactScopedFlags are the flags buildGeneratePlans reads only while +// building one kind of artifact. Given for the other kind they were accepted +// and dropped, so `--dataset --trace-days 7` produced a dataset and said +// nothing about the seven days. +var artifactScopedFlags = []struct { + name string + forEval bool // read under req.evaluator rather than req.dataset + otherFor string +}{ + {name: "from", otherFor: "--evaluator"}, + {name: "max-samples", otherFor: "--evaluator"}, + {name: "dataset-name", otherFor: "--evaluator"}, + {name: "trace-days", forEval: true, otherFor: "--dataset"}, + {name: "evaluator-name", forEval: true, otherFor: "--dataset"}, +} + +func refuseInapplicableFlags(cmd *cobra.Command, dataset, evaluator bool) error { + for _, f := range artifactScopedFlags { + applies := dataset + if f.forEval { + applies = evaluator + } + if !applies && cmd.Flags().Changed(f.name) { + return messages.FlagDoesNotApply(f.name, f.otherFor) + } + } + return nil +} + +// buildGeneratePlans settles everything that does not need the network, for +// each artifact asked for. Ordered dataset first, which is the order their +// progress is replayed in. +func buildGeneratePlans(req generateRequest) ([]generationPlan, error) { + plans := make([]generationPlan, 0, 2) + + if req.dataset { + name, err := generatedName(req.datasetName, req.target, "dataset") + if err != nil { + return nil, err + } + plan, err := resolvePlan(req.flags, name, project.DefaultDatasetsDir) + if err != nil { + return nil, err + } + plan.Kind = generateKindDataset + plan.From = req.from + plan.SampleSize = req.maxSamples + if plan.SampleSize == 0 { + plan.SampleSize = project.DefaultSampleSize + } + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), + req.flags.force, + ); err != nil { + return nil, err + } + plans = append(plans, plan) + } + + if req.evaluator { + name, err := generatedName(req.evaluatorName, req.target, "evaluator") + if err != nil { + return nil, err + } + plan, err := resolvePlan(req.flags, name, project.DefaultEvaluatorsDir) + if err != nil { + return nil, err + } + plan.Kind = generateKindEvaluator + plan.TraceDays = req.traceDays + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), + req.flags.force, + ); err != nil { + return nil, err + } + plans = append(plans, plan) + } + + return plans, nil +} + +// generatedName is the explicit name, or one derived from the target. +// +// The name becomes a filename as well as a service asset name, so it is +// checked here: `--dataset-name ../../x` would otherwise write outside the +// directory the caller pointed generation at, and `--force` would overwrite +// whatever is there. +func generatedName(explicit, target, suffix string) (string, error) { + name := explicit + if name == "" { + if target == "" { + return "", messages.GeneratedNameNeedsATarget(suffix) + } + name = target + "-" + suffix + } + if !nameIsAPathComponent(name) { + return "", messages.GeneratedNameNotAFileName(suffix, name) + } + return name, nil +} + +// nameIsAPathComponent reports whether a name stays where it is put. +// +// Only the filesystem's objections are checked. The service enforces its own +// character set, and duplicating it here would refuse names it accepts. +func nameIsAPathComponent(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + // A leading dash writes a file whose name reads as a flag to whatever the + // caller pipes the path into, which is the filesystem's objection rather + // than the service's. + if strings.HasPrefix(name, "-") { + return false + } + if strings.ContainsAny(name, `/\:`) || filepath.IsAbs(name) { + return false + } + return true +} + +type generationOutcome struct { + plan generationPlan + ref *project.ArtifactRef + jobID string + output bytes.Buffer + err error +} + +// runGenerations submits every plan at once and settles them together. +func (ec *evalContext) runGenerations( + cmd *cobra.Command, + plans []generationPlan, + flags generateFlags, +) error { + outcomes := make([]generationOutcome, len(plans)) + var wg sync.WaitGroup + + // The announcements go out before the goroutines start, so a long + // generation is not silent while it runs. Only the per-job progress is + // buffered, which is what would interleave. + out := cmd.OutOrStdout() + if !isJSON(cmd) { + for i := range plans { + fmt.Fprint(out, messages.GenerationStarting(string(plans[i].Kind), plans[i].Name)) + } + } + + for i := range plans { + outcomes[i].plan = plans[i] + wg.Add(1) + go func(o *generationOutcome) { + defer wg.Done() + switch o.plan.Kind { + case generateKindDataset: + o.ref, o.err = ec.generateDataset( + cmd.Context(), o.plan, &o.output, flags.noWait, &o.jobID) + default: + o.ref, o.err = ec.generateRubric( + cmd.Context(), o.plan, &o.output, flags.noWait, &o.jobID) + } + }(&outcomes[i]) + } + wg.Wait() + + // A failed write must not cost the caller the catalog entries for work the + // service already billed them for, so it is carried rather than returned. + var failures []error + if !isJSON(cmd) { + for i := range outcomes { + if _, err := out.Write(outcomes[i].output.Bytes()); err != nil { + failures = append(failures, err) + break + } + } + } + + // Catalog entries land in one file, so they are written here rather than + // from the goroutines that produced them. + for i := range outcomes { + o := &outcomes[i] + if o.err != nil { + failures = append(failures, messages.GenerationFailed(string(o.plan.Kind), o.err)) + continue + } + var err error + switch o.plan.Kind { + case generateKindDataset: + err = addDatasetToCatalog(cmd, flags.path, o.ref) + default: + err = addEvaluatorToCatalog(cmd, flags.path, o.ref) + } + if err != nil { + failures = append(failures, err) + } + } + + if len(failures) > 0 { + return messages.SomeGenerationsFailed(failures) + } + + // One document, keyed by artifact: two bare objects on stdout is not + // something a caller can parse. Under --no-wait there is no artifact yet, + // so the job id is what the caller gets and what they reattach with. + if isJSON(cmd) { + produced := map[string]any{} + for i := range outcomes { + o := &outcomes[i] + if o.ref != nil { + produced[string(o.plan.Kind)] = o.ref + continue + } + if o.jobID != "" { + produced[string(o.plan.Kind)] = map[string]string{"job_id": o.jobID} + continue + } + produced[string(o.plan.Kind)] = nil + } + return emitJSON(out, produced) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go new file mode 100644 index 00000000000..5ea3ec5ae42 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Omitting both flags is the zero-to-first-eval path the composite exists for, +// so it has to mean both rather than nothing. +func TestSelectedArtifacts(t *testing.T) { + cases := []struct { + name string + dataset, evaluator bool + wantDataset, wantEvaluator bool + }{ + {"neither means both", false, false, true, true}, + {"--dataset narrows", true, false, true, false}, + {"--evaluator narrows", false, true, false, true}, + {"both means both", true, true, true, true}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotDataset, gotEvaluator := selectedArtifacts(c.dataset, c.evaluator) + + assert.Equal(t, c.wantDataset, gotDataset, "dataset") + assert.Equal(t, c.wantEvaluator, gotEvaluator, "evaluator") + }) + } +} + +// The spec's defaults. Deriving from the target is what lets `generate` take no +// positional argument at all. +func TestGeneratedName_DerivesFromTheTarget(t *testing.T) { + name, err := generatedName("", "support-agent", "dataset") + require.NoError(t, err) + assert.Equal(t, "support-agent-dataset", name) + + name, err = generatedName("", "support-agent", "evaluator") + require.NoError(t, err) + assert.Equal(t, "support-agent-evaluator", name) +} + +func TestGeneratedName_ExplicitWins(t *testing.T) { + name, err := generatedName("golden", "support-agent", "dataset") + + require.NoError(t, err) + assert.Equal(t, "golden", name) +} + +// With neither there is nothing to name the artifact after, and the refusal has +// to name both flags that would answer it. +func TestGeneratedName_NeedsSomethingToNameItAfter(t *testing.T) { + _, err := generatedName("", "", "dataset") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--dataset-name") + assert.Contains(t, err.Error(), "--target") +} + +// A composite that submits two jobs has to build a plan for each. +func TestBuildGeneratePlans_BuildsBothPlans(t *testing.T) { + plans, err := buildGeneratePlans(generateRequest{ + flags: &generateFlags{path: t.TempDir(), target: "support-agent"}, + target: "support-agent", + dataset: true, + evaluator: true, + }) + + require.NoError(t, err) + require.Len(t, plans, 2) + assert.Equal(t, generateKindDataset, plans[0].Kind, + "dataset first, which is the order its progress is replayed in") + assert.Equal(t, generateKindEvaluator, plans[1].Kind) + assert.Equal(t, "support-agent-dataset", plans[0].Name) + assert.Equal(t, "support-agent-evaluator", plans[1].Name) +} + +// Narrowing builds one plan, so nothing is submitted for the other. +func TestBuildGeneratePlans_NarrowedToOne(t *testing.T) { + plans, err := buildGeneratePlans(generateRequest{ + flags: &generateFlags{path: t.TempDir(), target: "support-agent"}, + target: "support-agent", + dataset: true, + }) + + require.NoError(t, err) + require.Len(t, plans, 1) + assert.Equal(t, generateKindDataset, plans[0].Kind) +} + +// The name becomes a filename, so one carrying a separator would write outside +// the directory generation was pointed at, and --force would overwrite it. +func TestGeneratedName_RefusesANameThatWouldLeaveTheDirectory(t *testing.T) { + escapes := []string{ + "../outside", + "..\\outside", + "sub/dir", + "sub\\dir", + "..", + ".", + "C:\\Windows\\System32\\drivers\\etc\\hosts", + "/etc/passwd", + } + + for _, name := range escapes { + t.Run(name, func(t *testing.T) { + _, err := generatedName(name, "support-agent", "dataset") + + require.Errorf(t, err, "%q must not be accepted as a file name", name) + assert.Contains(t, err.Error(), "file name") + }) + } +} + +// The service decides its own character set. Refusing everything it might +// accept would block names that work. +func TestGeneratedName_AllowsOrdinaryNames(t *testing.T) { + for _, name := range []string{ + "golden", + "support-agent-dataset", + "support_agent.v2", + "caf\u00e9-dataset", + "dataset 2", + } { + t.Run(name, func(t *testing.T) { + got, err := generatedName(name, "support-agent", "dataset") + + require.NoError(t, err) + assert.Equal(t, name, got) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go new file mode 100644 index 00000000000..6445937d1b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func runGenerate(t *testing.T, args ...string) error { + t.Helper() + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(append([]string{"generate"}, args...)) + return root.ExecuteContext(context.Background()) +} + +// Each of these is read while building one kind of artifact and ignored while +// building the other, so given for the wrong one they were accepted and +// dropped -- `--evaluator --max-samples 50` produced a rubric and said nothing +// about the 50, and `--dataset --trace-days 7` a dataset and nothing about the +// seven days. +func TestFlagsThatCannotApplyAreRefused(t *testing.T) { + cases := []struct { + flag string + narrowed string + args []string + }{ + {"--from", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--from", "prompt"}}, + {"--max-samples", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--max-samples", "50"}}, + {"--dataset-name", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--dataset-name", "ds"}}, + {"--trace-days", "--dataset", + []string{"--dataset", "--dataset-name", "ds", "--trace-days", "7"}}, + {"--evaluator-name", "--dataset", + []string{"--dataset", "--dataset-name", "ds", "--evaluator-name", "ev"}}, + } + + for _, c := range cases { + err := runGenerate(t, c.args...) + + require.Errorf(t, err, "%s cannot apply under %s", c.flag, c.narrowed) + assert.Contains(t, err.Error(), c.flag) + assert.Containsf(t, err.Error(), c.narrowed, + "the refusal has to name the flag that made %s inapplicable", c.flag) + } +} + +// Generating both is the default, and every one of those flags applies then. +// Without this the guard above could be satisfied by refusing them always. +func TestNoFlagIsRefusedWhenBothArtifactsAreGenerated(t *testing.T) { + err := runGenerate(t, + "--dataset-name", "ds", "--evaluator-name", "ev", + "--from", "prompt", "--max-samples", "50", "--trace-days", "7") + + if err != nil { + assert.NotContains(t, err.Error(), "has no effect on what", + "generating both artifacts makes every one of these flags applicable") + } +} + +// And each flag is still accepted for the artifact it does affect. +func TestEachFlagIsAcceptedForItsOwnArtifact(t *testing.T) { + forDataset := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--from", "prompt", "--max-samples", "50") + if forDataset != nil { + assert.NotContains(t, forDataset.Error(), "has no effect on what") + } + + forEvaluator := runGenerate(t, + "--evaluator", "--evaluator-name", "ev", "--trace-days", "7") + if forEvaluator != nil { + assert.NotContains(t, forEvaluator.Error(), "has no effect on what") + } +} + +// Zero already means "seed the rubric from no traces", so a negative window has +// nothing left to mean. It was accepted and read as zero, quietly producing a +// rubric with none of the trace seeding that was asked for. +func TestNegativeTraceDaysIsRefused(t *testing.T) { + err := runGenerate(t, "--evaluator", "--evaluator-name", "ev", "--trace-days", "-5") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--trace-days") + assert.Contains(t, err.Error(), "-5", "the value that was rejected") +} + +// Zero is a real answer and has to keep working. +func TestZeroTraceDaysIsAccepted(t *testing.T) { + err := runGenerate(t, "--evaluator", "--evaluator-name", "ev", "--trace-days", "0") + + if err != nil { + assert.NotContains(t, err.Error(), "--trace-days", + "0 is how a caller says to read no traces") + } +} + +// --no-wait returns as soon as the job is submitted, so there is no artifact to +// place. Accepting both left the caller waiting for a file never coming. +func TestOutputDirWithNoWaitIsRefused(t *testing.T) { + err := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--no-wait", "--output-dir", t.TempDir()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--output-dir") + assert.Contains(t, err.Error(), "--no-wait") + assert.Contains(t, err.Error(), "job show", + "the refusal has to name how to collect the artifact later") +} + +// An output directory without --no-wait is what the flag is for. +func TestOutputDirAloneIsStillAccepted(t *testing.T) { + err := runGenerate(t, "--dataset", "--dataset-name", "ds", "--output-dir", t.TempDir()) + + if err != nil { + assert.NotContains(t, err.Error(), "nothing to write to", + "an output directory without --no-wait must not be refused") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go new file mode 100644 index 00000000000..26a96aad84e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// `generate` decides what to submit before it touches the network, so the plan +// it builds — which agent, what model, where the artifact lands — is checkable +// without paying for a generation job. These are the parts that cannot be +// observed afterwards: once the job is submitted, a wrong default is +// indistinguishable from an intended one. + +// evalsDir returns flags pointing at an empty eval directory. +func evalsDir(t *testing.T) *generateFlags { + t.Helper() + return &generateFlags{path: t.TempDir()} +} + +// withEvals writes a configuration into the flags' directory. +func withEvals(t *testing.T, f *generateFlags, evals ...project.Eval) { + t.Helper() + require.NoError(t, project.SaveEvalConfig(f.path, &project.EvalConfig{Evals: evals})) +} + +// Generation settings are flags only: there is no generate.yaml, because the +// artifact is checked in and regeneration usually wants different settings. +func TestResolvePlan_FromFlagsAlone(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + plan, err := resolvePlan(f, "shop-golden", project.DefaultDatasetsDir) + require.NoError(t, err) + + require.Equal(t, "shop-golden", plan.Name) + require.Equal(t, "shop-agent", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) + require.Equal(t, f.path, plan.BaseDir) +} + +// Each generate has its own default output directory, so a rubric never lands +// in the datasets folder. +func TestResolvePlan_OutputDirDefaultsPerArtifact(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + ds, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultDatasetsDir, ds.OutputDir) + + ev, err := resolvePlan(f, "r", project.DefaultEvaluatorsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) + + f.outputDir = "./from-flag" + override, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./from-flag", override.OutputDir) +} + +// A named target carries a deployment, and the spec makes it the default, so +// the plan settles without one and lets prepareGeneration read it. Refusing +// here would ask the caller for something the project already knows. +func TestResolvePlan_DefersToTheAgentForTheModel(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Empty(t, plan.Model, "the deployment is read from the agent, not guessed here") + require.Equal(t, "shop-agent", plan.Agent) +} + +// With no target either there is nothing to read a deployment from, so the +// refusal happens before authentication and names the flag that supplies one. +func TestResolvePlan_RequiresAGenerationModelWithNoAgent(t *testing.T) { + f := evalsDir(t) + + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.Error(t, err) + require.Contains(t, err.Error(), "--generation-model") +} + +// An input the caller named and got wrong is reported ahead of one they simply +// left out. Both checks are local, so the only thing deciding which the user +// sees is the order they run in — and a missing instruction file is a typo the +// caller can act on, while the model has a documented default path. +func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.instructionFile = filepath.Join(t.TempDir(), "absent.md") + + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file", + "the flag the caller got wrong must win over the one they omitted") +} + +// The target is already declared on an eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheDeclaredTarget(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + withEvals(t, f, project.Eval{ + Name: "support-agent-eval", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + }) + + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "support-agent", plan.Agent, + "the declared target is the agent to generate from") + + // An explicit flag still wins, which is what makes a one-off run possible + // without editing a file that is checked in. + f.target = "from-flag" + plan, err = resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "from-flag", plan.Agent) +} + +// With no configuration at all, generation runs from the instruction alone. +// This is the golden path: both generates precede init. +func TestResolvePlan_NoConfigurationYet(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + f.instruction = "test refunds and returns" + + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Empty(t, plan.Agent) + require.Equal(t, "test refunds and returns", plan.Instruction) +} + +// The bounds are the service's, and the boundaries themselves have to be +// accepted: a check that rejected 15 or 1000 would be indistinguishable from +// one that is simply too strict. +func TestGenerateSampleSizeBounds(t *testing.T) { + for _, tc := range []struct { + size int + allowed bool + }{ + {project.MinSampleSize - 1, false}, + {project.MinSampleSize, true}, + {project.DefaultSampleSize, true}, + {project.MaxSampleSize, true}, + {project.MaxSampleSize + 1, false}, + } { + err := project.ValidateSampleSize(tc.size) + if tc.allowed { + require.NoErrorf(t, err, "%d is inside the service's range", tc.size) + continue + } + require.Errorf(t, err, "%d is outside the service's range", tc.size) + require.Contains(t, err.Error(), "must be between") + } +} + +func TestResolveInstruction(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(filled, []byte(" test refunds and returns\n\n"), 0o600)) + blank := filepath.Join(dir, "blank.md") + require.NoError(t, os.WriteFile(blank, []byte(" \n"), 0o600)) + + t.Run("inline is returned as given", func(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + }) + + t.Run("a file is read and trimmed", func(t *testing.T) { + got, err := resolveInstruction("", filled) + require.NoError(t, err) + require.Equal(t, "test refunds and returns", got) + }) + + // A whitespace-only file would otherwise generate from nothing, which + // produces a rubric with no relation to the agent. + t.Run("an empty file is refused", func(t *testing.T) { + _, err := resolveInstruction("", blank) + require.Error(t, err) + require.Contains(t, err.Error(), "is empty") + }) + + t.Run("a missing file names the flag", func(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(dir, "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go new file mode 100644 index 00000000000..c38c1d08f21 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generated rubric is the file the next deploy compares against, so a field +// dropped on the way to disk republishes the evaluator without it. pass_threshold +// is what decides pass or fail, so losing it changes grading silently. +func TestWriteRubricKeepsTheWholeDefinition(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + + result := json.RawMessage(`{ + "name": "support-agent-quality", + "version": "1", + "definition": { + "type": "rubric", + "pass_threshold": 0.5, + "dimensions": [{"id": "accuracy", "description": "Correct.", "weight": 9}], + "something_the_service_added_later": true + } + }`) + + require.NoError(t, writeRubric(path, result)) + + var got map[string]any + body, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &got)) + + assert.Equal(t, 0.5, got["pass_threshold"], + "the threshold decides pass or fail, so losing it changes grading") + assert.Equal(t, true, got["something_the_service_added_later"], + "the definition is written through, so a new field is not lost either") + assert.Equal(t, "rubric", got["type"]) + assert.Len(t, got["dimensions"], 1) + assert.NotContains(t, got, "name", "only the definition is written, not the envelope") +} + +// A payload that is not a rubric is kept verbatim rather than discarded. +func TestWriteRubricFallsBackToTheRawPayload(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, writeRubric(path, json.RawMessage(`{"unexpected":"shape"}`))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.JSONEq(t, `{"unexpected":"shape"}`, string(body)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go new file mode 100644 index 00000000000..ba8d5861714 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's default: traces when the project has Application Insights +// connected, otherwise the agent. The connection string is how a project says +// it collects traces at all, so asking for traces without one would submit a +// billed job against nothing. +func TestDefaultGenerationSource(t *testing.T) { + assert.Equal(t, []string{"traces"}, + defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000"), + "a project collecting traces should be generated from them") + + assert.Equal(t, []string{"agent"}, defaultGenerationSource(""), + "with nowhere for traces to have been collected, the agent is all there is") +} + +// --from is a request, and one the plan cannot honour has to stop the command +// rather than quietly submit a job seeded from less than was asked for. +func TestRefuseUnbuildableSources(t *testing.T) { + assert.NoError(t, refuseUnbuildableSources(nil)) + assert.NoError(t, refuseUnbuildableSources([]string{})) + + tests := []struct { + kind string + says string + }{ + {"prompt", "--agent-instruction"}, + {"agent", "--target"}, + {"file", "azd ai eval dataset create"}, + } + + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + err := refuseUnbuildableSources([]string{tt.kind}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.says, + "the error has to name the way out, not just the problem") + }) + } +} + +// Two unhonoured sources are two things the caller has to fix, so both are +// reported at once rather than one per attempt. +func TestRefuseUnbuildableSources_ReportsAllOfThemAtOnce(t *testing.T) { + err := refuseUnbuildableSources([]string{"prompt", "agent"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--agent-instruction") + assert.Contains(t, err.Error(), "--target") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go new file mode 100644 index 00000000000..e4c04baebf8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service's system error says only that something went wrong and to try +// again, but agent-seeded generation fails deterministically, so a bare retry +// suggestion sends users into a loop. +func TestExplainDataGenerationFailureAddsAgentContext(t *testing.T) { + err := errors.New( + `job failed with status "failed": Something went wrong during data generation. Please try again.`) + + explained := explainDataGenerationFailure(err, "my-agent") + require.Error(t, explained) + require.Contains(t, explained.Error(), "my-agent") + require.Contains(t, explained.Error(), "--dataset") + require.ErrorIs(t, explained, err, "the original error must stay in the chain") +} + +// The code spelling is matched as well, in case the poller starts surfacing it. +func TestExplainDataGenerationFailureMatchesErrorCode(t *testing.T) { + err := fmt.Errorf("job failed: DataGenerationJobSystemError") + explained := explainDataGenerationFailure(err, "my-agent") + require.Contains(t, explained.Error(), "Workarounds") +} + +// Unrelated failures are passed through untouched, and so is a job that had no +// agent source to blame. +func TestExplainDataGenerationFailureLeavesOthersAlone(t *testing.T) { + other := errors.New("submitting the data generation job: 403 Forbidden") + require.Equal(t, other, explainDataGenerationFailure(other, "my-agent")) + + systemErr := errors.New("Something went wrong during data generation") + require.Equal(t, systemErr, explainDataGenerationFailure(systemErr, "")) + + require.NoError(t, explainDataGenerationFailure(nil, "my-agent")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go new file mode 100644 index 00000000000..806812b409f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "net/http" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// digestIDKey is what makes a rename find the eval it already deployed: the id +// is recorded against the eval's substance, so a declaration whose name +// changed still resolves. That only works while the key derives from the +// digest the same way it did last deploy — change the format and every +// deployed eval silently loses its recorded id and gets recreated. +func TestDigestIDKey(t *testing.T) { + const digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + key := digestIDKey(digest) + + assert.Equal(t, "EVAL_SUBSTANCE_0123456789ABCDEF_ID", key) + for _, r := range key { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } +} + +// Same substance, same key — that is the whole mechanism. +func TestDigestIDKey_IsStableForTheSameSubstance(t *testing.T) { + group := project.Eval{ + Name: "support", + Dataset: "support-regression", + Target: &project.Target{Name: "support-agent"}, + } + + first, err := project.FingerprintGroup(group) + require.NoError(t, err) + + renamed := group + renamed.Name = "support-renamed" + renamed.Description = "reworded" + second, err := project.FingerprintGroup(renamed) + require.NoError(t, err) + + assert.Equal(t, digestIDKey(first), digestIDKey(second), + "a rename must land on the key the first deploy wrote") +} + +// Different substance, different key, so a genuinely new eval does not adopt +// an unrelated one's id. +func TestDigestIDKey_DiffersWhenTheSubstanceDoes(t *testing.T) { + a, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "one"}) + require.NoError(t, err) + b, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "two"}) + require.NoError(t, err) + + assert.NotEqual(t, digestIDKey(a), digestIDKey(b)) +} + +// The version recorded for an artifact comes out of what the service returned, +// falling back to what the caller already knew. +func TestVersionFromRaw(t *testing.T) { + tests := []struct { + name string + raw string + fallback string + want string + }{ + {"version in the body wins", `{"version":"7"}`, "3", "7"}, + {"empty version falls back", `{"version":""}`, "3", "3"}, + {"absent version falls back", `{"name":"x"}`, "3", "3"}, + {"unparseable body falls back", `not json`, "3", "3"}, + {"empty body falls back", ``, "3", "3"}, + {"no fallback either", `{}`, "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, versionFromRaw([]byte(tt.raw), tt.fallback)) + }) + } +} + +// A criterion binds to a dataset column through `{{item.}}`. Reading the +// name wrong is how a run is submitted against a column the dataset does not +// have, which the service rejects without saying which one. +func TestItemColumn(t *testing.T) { + bound := map[string]string{ + "{{item.query}}": "query", + "{{item.ground_truth}}": "ground_truth", + "{{item.a.b}}": "a.b", + } + for binding, want := range bound { + got, ok := itemColumn(binding) + assert.Truef(t, ok, "%q is a binding", binding) + assert.Equal(t, want, got) + } + + notBound := []string{ + "", + "query", + "{{item.}}", + "{{ item.query }}", + "{{item.query", + "item.query}}", + "{{response.output}}", + } + for _, binding := range notBound { + got, ok := itemColumn(binding) + assert.Falsef(t, ok, "%q is not an item binding", binding) + assert.Empty(t, got) + } +} + +// An eval is named after what it evaluates and what it reads, so two evals over +// the same agent from different sources do not collide. +func TestDefaultEvalName(t *testing.T) { + assert.Equal(t, "support-agent-trace-eval", + defaultEvalName("support-agent", initSourceTraces)) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "dataset")) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "")) + + assert.NotEqual(t, + defaultEvalName("support-agent", initSourceTraces), + defaultEvalName("support-agent", "dataset"), + "the source is in the name so the two do not collide") +} + +// The reattach line printed by --no-wait has to name the group the job +// actually belongs to; the two job types share no collection, so the wrong +// group is a command that returns "not found". +func TestJobLookupErrorNamesTheGroup(t *testing.T) { + for _, kind := range []jobKind{datasetJobs, evaluatorJobs} { + err := jobLookupError("reading", kind, "job_1", assert.AnError) + + require.Error(t, err) + assert.Contains(t, err.Error(), "job_1") + assert.Truef(t, strings.Contains(err.Error(), kind.name), + "the error must name the %q group so the retry goes to the right one", kind.name) + } +} + +// A failed delete used to report that a read failed, which sends the reader +// looking for a read that never happened. +func TestJobLookupErrorNamesWhatWasAttempted(t *testing.T) { + deleteErr := jobLookupError("deleting", datasetJobs, "job_1", assert.AnError) + require.Error(t, deleteErr) + assert.Contains(t, deleteErr.Error(), "deleting") + assert.NotContains(t, deleteErr.Error(), "reading") + + cancelErr := jobLookupError("cancelling", datasetJobs, "job_1", assert.AnError) + require.Error(t, cancelErr) + assert.Contains(t, cancelErr.Error(), "cancelling") + + // A genuine 404 still points at the sibling group whatever the verb was. + notFound := jobLookupError("deleting", datasetJobs, "job_1", + &azcore.ResponseError{StatusCode: http.StatusNotFound}) + require.Error(t, notFound) + assert.Contains(t, notFound.Error(), jobKindEvaluator) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go new file mode 100644 index 00000000000..0182326e691 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -0,0 +1,847 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + "sync" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" + "google.golang.org/protobuf/types/known/structpb" +) + +// Data sources `init` can point an eval at. +const ( + initSourceDataset = "dataset" + initSourceTraces = "traces" +) + +// newInitCommand scaffolds the eval configuration. It makes no service calls at +// all, so it works offline and unauthenticated. +// +// It only ever adds. A name already declared is refused rather than +// overwritten, because the settings a reader tunes by hand — thresholds, judge +// model, data mapping — live nowhere but that entry and `init` cannot +// reproduce them. Editing an eval is a file edit. +func newInitCommand() *cobra.Command { + var ( + evalName string + target string + source string + dataset string + maxTraces int + evaluators []string + judgeModel string + path string + force bool + ) + + cmd := &cobra.Command{ + Use: "init", + Short: "Scaffold evaluation config for an agent. Makes no service calls.", + RunE: func(cmd *cobra.Command, args []string) error { + out := cmd.OutOrStdout() + + switch source { + case "", initSourceDataset, initSourceTraces: + default: + return messages.SourceNotADataSource( + source, initSourceDataset, initSourceTraces) + } + if source == initSourceTraces && dataset != "" { + return messages.TracesTakesNoDataset() + } + if cmd.Flags().Changed("max-traces") && source != initSourceTraces { + return messages.MaxTracesNeedsTraceSource() + } + if maxTraces < 0 { + return messages.MaxTracesMustBePositive() + } + // Checked with the other flag-only rules, before anything is asked + // or read: a reference that cannot name an evaluator is otherwise + // written by a command that exits 0, and only fails two commands + // later. Answering two prompts first to be told a flag was wrong is + // the same defect one step removed. + if err := validateEvaluatorRefs(evaluators); err != nil { + return err + } + // Asked twice -- once to pick the default source, once to say so -- + // and each call opens an azd connection. The answer cannot change + // mid-command, and a run that never asks never connects. + tracesWired := sync.OnceValue(func() bool { + return tracesConnected(commandContext(cmd)) + }) + if source == "" { + // The same signal `generate --from` defaults on, read from the azd + // environment rather than the service, so init still makes no + // service calls. Traces are real conversations; a project wired to + // collect them should not have to ask for them by flag. + if tracesWired() { + source = initSourceTraces + } else { + source = initSourceDataset + } + } + // The same cascade every other command reads the configuration + // through. init merges into the configuration it finds, so a second + // `init` in a project scaffolded at ./quality has to find that one -- + // otherwise it writes a second configuration under ./evals and + // declares a second service pointing at it. + path, err := resolveEvalDir(cmd.Context(), path) + if err != nil { + return err + } + + // Asked before anything is written: the project is the one thing + // init cannot supply for itself, and failing after creating + // directories leaves a half-scaffolded tree behind. + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { + return err + } + + // The target is what the whole scaffold is named and shaped around, + // so it is settled before anything derived from it. + if target == "" { + target, err = resolveAgentTarget(cmd, azdProject) + if err != nil { + return err + } + } + if evalName == "" { + evalName = defaultEvalName(target, source) + } + if judgeModel == "" { + judgeModel, err = resolveJudgeModel(cmd, azdProject) + if err != nil { + return err + } + } + + configPath, err := project.ResolveEvalConfigPath(path) + if err != nil { + return err + } + // Captured before the write: init merges into an existing config, so + // reporting it as created would claim a file it only added to. + _, configExistedErr := os.Stat(configPath) + configExisted := configExistedErr == nil + cfg, err := project.OpenEvalConfig(path) + if err != nil { + return err + } + if cfg == nil { + cfg = &project.EvalConfig{} + } + // Checked before the prompt as well as after it, so a name that is + // already taken is reported without asking a question first. + if cfg.HasEval(evalName) && !force { + return messages.EvalAlreadyDeclared( + evalName, filepath.ToSlash(configPath)) + } + + // Asked, not detected: an eval grades on a set, so there is no + // "the only one" to settle on, and which criteria define quality + // is the substantive decision in the configuration. + // + // Deliberately outside the lock below. This is an unbounded human + // pause, and a lock held across it would either block a concurrent + // `generate` for as long as someone leaves the terminal, or -- once + // that side gave up waiting -- protect nothing at all. The listing + // it offers is only a menu; the authoritative read is taken after. + evaluatorsWereChosen := len(evaluators) > 0 + if len(evaluators) == 0 { + var asked bool + evaluators, asked, err = resolveEvaluators( + cmd, cfg, target+"-quality", source == initSourceTraces) + if err != nil { + return err + } + evaluatorsWereChosen = asked + } + + // The read-modify-write starts here, and nothing inside it waits on + // a person. The configuration is read again because the copy above + // was taken before the prompt, and a `generate` may well have + // finished writing to it since. + unlockConfig, err := project.LockEvalConfig(cmd.Context(), path) + if err != nil { + return err + } + defer unlockConfig() + + cfg, err = project.OpenEvalConfig(path) + if err != nil { + return err + } + if cfg == nil { + cfg = &project.EvalConfig{} + } + if cfg.HasEval(evalName) { + if !force { + return messages.EvalAlreadyDeclared( + evalName, filepath.ToSlash(configPath)) + } + cfg.RemoveEval(evalName) + } + + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { + return messages.CreatingDatasetsDir(err) + } + if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { + return messages.CreatingEvaluatorsDir(err) + } + + plan := planScaffold(scaffoldInput{ + evalName: evalName, + target: target, + source: source, + dataset: dataset, + maxTraces: maxTraces, + evaluators: evaluators, + judgeModel: judgeModel, + rubricName: target + "-quality", + evalDir: path, + cfg: cfg, + }) + + if err := project.SaveEvalConfig(path, cfg); err != nil { + return err + } + + // Scaffolding a config azd cannot see is half a step: the eval + // service has to be referenced from the root config before any of + // `azd up`, `azd deploy` or `azd ai eval run` will act on it. + serviceName := target + "-evals" + rootWiring, err := ensureRootEvalService(cmd.Context(), serviceName, target, configPath) + if err != nil { + return err + } + + recordEvalPath(cmd.Context(), path) + + if isJSON(cmd) { + return emitJSON(out, map[string]any{ + "eval": evalName, + "evalConfig": configPath, + "service": serviceName, + "datasetsDir": filepath.Join(path, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "source": source, + "judgeModel": judgeModel, + "evaluators": plan.evaluatorNames(), + }) + } + + fmt.Fprint(out, messages.DetectedTarget(target)) + if source == initSourceTraces { + // Claiming the connection is only honest when it was found. init + // makes no service calls, so it cannot verify one it did not see. + fmt.Fprint(out, messages.UsingTraceSource(tracesWired())) + } + // Only what was settled without asking: a reader who just picked + // from a list does not need it read back to them. + if names := plan.evaluatorNames(); len(names) > 0 && !evaluatorsWereChosen { + fmt.Fprint(out, messages.GradingWith(names)) + } + if judgeModel != "" { + fmt.Fprint(out, messages.JudgeModelDeployment(judgeModel)) + } + + fmt.Fprint(out, messages.ScaffoldHeading(configExisted)) + fmt.Fprint(out, messages.ScaffoldConfigLine(filepath.ToSlash(configPath), configExisted)) + switch rootWiring { + case wiringAdded: + fmt.Fprint(out, messages.AddedServiceLine(rootConfigName, serviceName)) + case wiringPresent: + fmt.Fprint(out, messages.AlreadyDeclaresServiceLine(rootConfigName, serviceName)) + } + + // Only what was actually scheduled is offered. Suggesting + // `dataset generate` for a dataset the caller supplied sends them + // to submit a billed job for an artifact they already have. + next := plan.nextSteps(deployCommandName(azdProject)) + fmt.Fprint(out, messages.FirstNextStep(next[0])) + for _, step := range next[1:] { + fmt.Fprint(out, messages.FurtherNextStep(step)) + } + return nil + }, + } + + cmd.Flags().StringVar(&evalName, "name", "", + "Name of the eval. Defaults to -eval, or -trace-eval under --source traces.") + cmd.Flags().StringVar(&target, "target", "", + "Name of the agent to evaluate. Detected when the project has one agent; prompts when it has several.") + cmd.Flags().StringVar(&source, "source", "", + "Where rows come from: dataset or traces. Defaults to traces when the azd "+ + "environment records an Application Insights connection, otherwise dataset.") + cmd.Flags().StringVar(&dataset, "dataset", "", + "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, + "Cap on traces read by a --source traces eval. Delete max_traces from the "+ + "file to take the service default instead.") + cmd.Flags().StringSliceVar(&evaluators, "evaluator", nil, + "Evaluator reference, repeatable and comma-separated. Use builtin. for a "+ + "built-in. Passing this replaces the defaults, so it also opts out of rubric generation.") + cmd.Flags().StringVar(&judgeModel, "judge-model", "", + "Model deployment the graders judge with. Detected from the project when omitted.") + cmd.Flags().StringVar(&path, "path", "", + "Directory to write the configuration into. Used verbatim, never re-rooted. "+ + "Defaults to the directory an earlier `init` scaffolded, otherwise ./evals.") + cmd.Flags().BoolVar(&force, "force", false, + "Replace an eval of the same name instead of failing.") + return cmd +} + +// defaultEvalName names an eval after what it evaluates and what it reads. +func defaultEvalName(target, source string) string { + if source == initSourceTraces { + return target + "-trace-eval" + } + return target + "-eval" +} + +// scaffoldInput is everything planScaffold needs, gathered so the signature +// does not grow a seventh positional string. +type scaffoldInput struct { + evalName string + target string + source string + dataset string + maxTraces int + evaluators []string + judgeModel string + rubricName string + evalDir string + cfg *project.EvalConfig +} + +// scaffold is what `init` added, and what it should suggest doing next. +type scaffold struct { + eval *project.Eval + datasetName string + rubricName string + target string + judgeModel string + // evalDir is where the configuration was written, so the next steps can + // name it when it is not the default. + evalDir string + generateDataset bool + generateRubric bool +} + +// planScaffold appends one eval to the configuration, adding any catalog +// entries it needs. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold(in scaffoldInput) scaffold { + cfg := in.cfg + out := scaffold{ + rubricName: in.rubricName, + target: in.target, + judgeModel: in.judgeModel, + evalDir: in.evalDir, + } + + eval := project.Eval{ + Name: in.evalName, + Description: fmt.Sprintf("Basic quality evaluation for %s", in.target), + EvaluationLevel: project.EvaluationLevelTurn, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: in.target, + }, + } + + if in.source == initSourceTraces { + // A trace-backed eval filters by agent rather than invoking one: the + // conversations already happened. + eval.Target = nil + eval.Source = &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: in.target, + MaxTraces: in.maxTraces, + } + } else { + datasetName := in.evalName + datasetSource := "" + out.generateDataset = true + if in.dataset != "" { + out.generateDataset = false + if looksLikeLocalDataset(in.dataset) { + // --dataset is given relative to where the user is standing, + // but source: resolves relative to the config, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(in.dataset, in.evalDir) + datasetName = strings.TrimSuffix( + filepath.Base(in.dataset), filepath.Ext(in.dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = in.dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + eval.Dataset = datasetName + out.datasetName = datasetName + addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource}) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: judging built-ins declare it as required, so an + // eval that leaves it off is rejected before it runs. The binding step + // drops it again for a rule-based evaluator that declares no judge. + initParams := map[string]any{} + if in.judgeModel != "" { + initParams["model"] = in.judgeModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + maps.Copy(params, initParams) + ref.InitializationParameters = params + return ref + } + + refs := evalcore.EvaluatorList{} + if len(in.evaluators) == 0 { + refs = append(refs, + withModel(evalcore.EvaluatorRef{ + Evaluator: evalcore.BuiltinPrefix + "task_adherence", + })) + if in.source != initSourceTraces { + refs = append(refs, withModel(evalcore.EvaluatorRef{Evaluator: in.rubricName})) + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: in.rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, in.rubricName), + }) + out.generateRubric = true + } + } else { + for _, e := range in.evaluators { + ref := evalcore.EvaluatorRef{Evaluator: e} + refs = append(refs, withModel(ref)) + if ref.IsBuiltin() { + continue + } + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + // Chosen, not defaulted, but it is still the rubric init offers to + // write, so it still has to be generated. Without this the config + // declares a file that nothing produces and `create` fails looking + // for it. + if e == in.rubricName { + out.generateRubric = true + } + } + } + eval.Evaluators = refs + + cfg.Evals = append(cfg.Evals, eval) + out.eval = &cfg.Evals[len(cfg.Evals)-1] + return out +} + +// addDatasetDecl adds a catalog entry unless the name is already declared. +func addDatasetDecl(cfg *project.EvalConfig, decl project.DatasetDecl) { + if decl.Name == "" { + return + } + // A source-less entry is still declared: it names a dataset already + // registered on the project. Skipping it left the eval referencing a + // dataset absent from the catalog, which its own validation rejects. + if _, ok := cfg.DatasetDeclaration(decl.Name); ok { + return + } + cfg.Datasets = append(cfg.Datasets, decl) +} + +// addEvaluatorDecl adds a catalog entry unless the name is already declared. +func addEvaluatorDecl(cfg *project.EvalConfig, decl project.EvaluatorDecl) { + if _, ok := cfg.EvaluatorDeclaration(decl.Name); ok { + return + } + cfg.Evaluators = append(cfg.Evaluators, decl) +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Evaluator) + } + return names +} + +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. +// +// Every generate step carries --target and --generation-model, which `generate` +// requires and does not detect. Omitting them printed a next step that failed +// twice before it ran, each failure naming one more flag. +func (s scaffold) nextSteps(deployCmd string) []string { + var steps []string + switch { + case s.generateDataset && s.generateRubric: + // One command produces both, which is the whole point of the composite. + steps = append(steps, s.generateCommand("")) + case s.generateDataset: + steps = append(steps, s.generateCommand("--dataset --dataset-name "+s.datasetName)) + case s.generateRubric: + steps = append(steps, s.generateCommand("--evaluator --evaluator-name "+s.rubricName)) + } + if len(steps) == 0 { + // `azd up` reads azure.yaml, which already $refs the configuration + // wherever it was written, so it is the one step --path must not join. + deploy := deployCmd + if deploy != azdUpCommand { + deploy = s.withPath(deploy) + } + steps = append(steps, deploy, s.withPath("azd ai eval run start")) + } + return steps +} + +// withPath appends --path to a step that needs it to run where init wrote. +// +// The recorded EVAL_CONFIG_PATH would usually supply this on its own, but +// recording it is best effort -- it needs an azd environment, and `init` works +// without one. Naming the directory makes the printed step run as printed +// either way, which is the claim these lines make. +func (s scaffold) withPath(step string) string { + if s.evalDir == "" || s.evalDir == project.DefaultEvalDir { + return step + } + return step + " --path " + quoteForShell(s.evalDir) +} + +// quoteForShell wraps a value a shell would otherwise read as more than one +// argument. +// +// `--path "./team evals"` is the difference between a printed step that runs +// and one that resolves ./team and reports the configuration missing. That is +// the case worth getting right, and double quotes are what cmd, PowerShell, +// bash and zsh all read the same way for a path containing spaces. +// +// A path containing $ or a backtick has no portable answer: double quotes stop +// neither from expanding in bash, zsh or PowerShell, and single quotes -- which +// would -- are literal only on the POSIX shells. Such a path is wrapped anyway, +// because one argument that may expand still beats two that certainly break, +// and this line is printed for a person to read rather than executed here. +// +// Backslashes are left alone: C:\Users\Me\My Evals has to come back out as +// itself, and doubling them would be right for bash and wrong for the two +// shells most likely to be reading a path that looks like that. +func quoteForShell(v string) string { + if v == "" { + return `""` + } + if !strings.ContainsAny(v, " \t\n\"'`$&|;<>()*?[]#~!") { + return v + } + return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` +} + +// generateCommand builds a `generate` invocation that runs as printed. +func (s scaffold) generateCommand(what string) string { + cmd := "azd ai eval generate" + if what != "" { + cmd += " " + what + } + if s.target != "" { + cmd += " --target " + s.target + } + if s.judgeModel != "" { + cmd += " --generation-model " + s.judgeModel + } + return s.withPath(cmd) +} + +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the eval config. +func relativeToConfig(path, evalDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(evalDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + +// rootConfigName is azd's project file, which the eval service is declared in. +const rootConfigName = "azure.yaml" + +// aiProjectHost is the Foundry project service other extensions declare. The +// eval service uses it for ordering when the repo has one. +const aiProjectHost = "azure.ai.project" + +// How the root config ended up referencing the eval service. +const ( + wiringAdded = "added" // the service was added to the project + wiringPresent = "present" // an eval service was already declared +) + +// readAzdProject returns the project, without changing it. +func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, messages.NoAzdProject() + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return nil, messages.NoAzdProject() + } + return resp.GetProject(), nil +} + +// azdDefaultInfraDir is where azd looks for infrastructure when the project +// does not name a directory itself. +const azdDefaultInfraDir = "infra" + +// projectCanProvision reports whether `azd provision` has anything to compile. +// +// This mirrors the provider sniff in azd's detectProviderFromFiles: the +// provider is inferred from the files in the infra directory, and a missing +// directory leaves it unspecified, which falls back to Bicep and fails on the +// absent infra/main.bicep -- verified against azd 1.30.0, where `azd up` on an +// eval-only project exits 1 and `azd deploy` does not run either. +// +// It is deliberately only that sniff. azd's real decision, ProjectInfrastructure, +// is also satisfied by infra layers, a .NET Aspire AppHost, and a `resources:` +// block in azure.yaml, none of which look at this directory. Each makes this +// answer false where `azd up` would have worked, so the cost of being wrong is +// naming our own command in a project that could also have provisioned -- which +// still publishes the eval. +func projectCanProvision(proj *azdext.ProjectConfig) bool { + if proj == nil { + return false + } + + dir := proj.GetInfra().GetPath() + if dir == "" { + dir = azdDefaultInfraDir + } + if !filepath.IsAbs(dir) { + dir = filepath.Join(proj.GetPath(), dir) + } + + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if e.IsDir() { + continue + } + switch filepath.Ext(e.Name()) { + case ".bicep", ".bicepparam", ".tf", ".tfvars": + return true + } + } + return false +} + +// aiModelHost is the model-deployment service the sibling Foundry extensions +// declare, which is where a judge deployment can be read without a service +// call. +const aiModelHost = "azure.ai.model" + +// detectModelDeployment finds the deployment the graders judge with, from what +// the project already declares. +// +// `init` makes no service calls, so detection is limited to the project file. +// Coming back empty leaves it to resolveJudgeModel, which reads the Foundry +// project's deployments: and then asks or names --judge-model. +func detectModelDeployment(proj *azdext.ProjectConfig) string { + for name, svc := range proj.GetServices() { + if svc.GetHost() != aiModelHost { + continue + } + if props := svc.GetAdditionalProperties().AsMap(); props != nil { + for _, key := range []string{"deployment", "deploymentName", "name", "model"} { + if v, ok := props[key].(string); ok && v != "" { + return v + } + } + } + return name + } + return "" +} + +// recordEvalPath remembers where the configuration was written, so the commands +// that read it afterwards do not need --path repeated. +// +// Best effort: `init` works outside an azd environment, and a path that could +// not be recorded only costs the caller a flag later. It is never a reason to +// fail a scaffold that already succeeded. +func recordEvalPath(ctx context.Context, path string) { + if path == "" || path == project.DefaultEvalDir { + return + } + azdClient, err := azdext.NewAzdClient() + if err != nil { + return + } + defer azdClient.Close() + + envName := azdEnvironmentName(ctx, azdClient) + if envName == "" { + return + } + _, _ = azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: envName, + Key: envKeyEvalPath, + Value: filepath.ToSlash(path), + }) +} + +// ensureRootEvalService declares the eval service in azd's project file. +// +// azd acts on nothing until the service exists, so the reference is made rather +// than described. It goes through azd's own Project().AddService, the same call +// the agents extension uses, so azd owns the edit and the project file keeps +// whatever shape azd gives it. +func ensureRootEvalService( + ctx context.Context, + serviceName, target, configPath string, +) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", messages.NoAzdProject() + } + + // A service already pointing at this configuration is left alone: + // re-adding it would deploy the same evals twice. + // + // Pointing at a different one is not the same thing. Matching on name and + // host alone reported the wiring present after `init --path` moved the + // configuration, and `azd up` went on deploying the file that was left + // behind -- the scaffold the reader was looking at was never deployed. + wantRef := "./" + filepath.ToSlash(configPath) + if svc, ok := resp.GetProject().GetServices()[serviceName]; ok && svc.GetHost() == project.EvalHost { + if have := serviceConfigRef(svc); have != "" && !sameRefTarget(have, wantRef) { + return "", messages.ServiceRefPointsElsewhere(serviceName, have, wantRef) + } + return wiringPresent, nil + } + + props, err := structpb.NewStruct(map[string]any{ + "$ref": wantRef, + }) + if err != nil { + return "", messages.BuildingServiceEntry(err) + } + + _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ + Service: &azdext.ServiceConfig{ + Name: serviceName, + Host: project.EvalHost, + Uses: evalServiceUses(resp.GetProject(), target), + AdditionalProperties: props, + }, + }) + if err != nil { + return "", messages.AddingServiceTo(rootConfigName, err) + } + return wiringAdded, nil +} + +// serviceConfigRef reads the $ref a service entry was authored with, or empty +// when it holds its configuration inline. +func serviceConfigRef(svc *azdext.ServiceConfig) string { + props := svc.GetAdditionalProperties() + if props == nil { + return "" + } + ref, _ := props.AsMap()["$ref"].(string) + return ref +} + +// sameRefTarget compares two $ref values as paths rather than as text, so +// `evals/azure.eval.yaml` and `./evals/azure.eval.yaml` are one answer. +func sameRefTarget(a, b string) bool { + return filepath.Clean(filepath.FromSlash(a)) == filepath.Clean(filepath.FromSlash(b)) +} + +// evalServiceUses orders the eval after the things it reads. +// +// It is conditional for the same reason the agents extension makes it +// conditional: naming a service the project does not declare is a broken +// reference, and an eval config can perfectly well sit in a repo that reaches +// an existing Foundry project by endpoint and an agent deployed elsewhere. +// +// Catalog entries need no ordering of their own — datasets, evaluators and +// evals are reconciled in a fixed order inside one deploy, forced by the +// contract rather than chosen. +func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { + var uses []string + for name, svc := range proj.GetServices() { + if svc.GetHost() == aiProjectHost { + uses = append(uses, name) + break + } + } + if _, ok := proj.GetServices()[target]; ok { + uses = append(uses, target) + } + return uses +} + +// looksLikeLocalDataset distinguishes a path from a registered dataset name. +func looksLikeLocalDataset(v string) bool { + if strings.ContainsAny(v, `/\`) { + return true + } + return strings.EqualFold(filepath.Ext(v), ".jsonl") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go new file mode 100644 index 00000000000..088189ee6eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "github.com/spf13/cobra" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// defaultEvaluators is what `init` proposes: one built-in that judges whether +// the agent did what was asked, plus a rubric generated from the agent's own +// instructions, which is what makes the criteria specific to this agent. +// +// The rubric is not offered for a trace-backed eval, which has no target to +// read instructions from. +func defaultEvaluators(rubricName string, traceBacked bool) []string { + refs := []string{evalcore.BuiltinPrefix + "task_adherence"} + if !traceBacked { + refs = append(refs, rubricName) + } + return refs +} + +// evaluatorChoices are the references `init` can offer. +// +// `init` makes no service calls, so the service's full built-in catalogue is +// not knowable here; offering a hardcoded copy of it would drift. What is +// knowable is the pair init proposes and whatever this configuration already +// declares. Anything else is reachable with --evaluator. +func evaluatorChoices(cfg *project.EvalConfig, rubricName string, traceBacked bool) []string { + seen := map[string]bool{} + var out []string + add := func(ref string) { + if ref == "" || seen[ref] { + return + } + seen[ref] = true + out = append(out, ref) + } + + for _, ref := range defaultEvaluators(rubricName, traceBacked) { + add(ref) + } + if cfg != nil { + for _, decl := range cfg.Evaluators { + add(decl.Name) + } + } + return out +} + +// resolveEvaluators settles what the eval grades on. +// +// Unlike the target and the judge model, there is no "the only one" here: an +// eval grades on a SET, and which criteria define quality for this agent is the +// substantive decision in the whole configuration. So this asks rather than +// detects, with the defaults preselected. Under --no-prompt the preselection +// stands, which is what keeps CI and the init -> generate flow working. +// +// The second return says whether the reader chose. Only a set decided FOR them +// is worth reporting back; echoing a selection they just made is noise. +func resolveEvaluators( + cmd *cobra.Command, + cfg *project.EvalConfig, + rubricName string, + traceBacked bool, +) ([]string, bool, error) { + defaults := defaultEvaluators(rubricName, traceBacked) + if noPrompt(cmd) { + return defaults, false, nil + } + chosen, err := promptEvaluators(cmd, evaluatorChoices(cfg, rubricName, traceBacked), defaults) + if err != nil { + return nil, false, err + } + return chosen, true, nil +} + +// promptEvaluators asks which references to grade with, defaults ticked. +func promptEvaluators(cmd *cobra.Command, choices, preselected []string) ([]string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + ticked := map[string]bool{} + for _, p := range preselected { + ticked[p] = true + } + opts := make([]*azdext.MultiSelectChoice, 0, len(choices)) + for i := range choices { + opts = append(opts, &azdext.MultiSelectChoice{ + Label: choices[i], + Value: choices[i], + Selected: ticked[choices[i]], + }) + } + + resp, err := azdClient.Prompt().MultiSelect(commandContext(cmd), &azdext.MultiSelectRequest{ + Options: &azdext.MultiSelectOptions{ + Message: messages.SelectEvaluatorsPrompt(), + Choices: opts, + }, + }) + if err != nil { + return nil, messages.SelectingEvaluators(err) + } + + chosen := make([]string, 0, len(resp.GetValues())) + for _, v := range resp.GetValues() { + // A blank choice would otherwise be written to the config as an + // evaluator named "", and looked up as one two commands later. + if v.GetValue() == "" { + continue + } + chosen = append(chosen, v.GetValue()) + } + if len(chosen) == 0 { + // An eval that grades on nothing is rejected by the service, and the + // refusal names none of this. + return nil, messages.NoEvaluatorsChosen() + } + return chosen, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go new file mode 100644 index 00000000000..e0e5fb929f1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// What an eval grades on is a SET, so there is no "the only one" to detect the +// way there is for the target and the judge model. Which criteria define +// quality is the substantive decision in the configuration, so init asks. +// +// The defaults are what it proposes, and they are what --no-prompt takes. +func TestDefaultEvaluatorsProposeABuiltinAndTheRubric(t *testing.T) { + assert.Equal(t, + []string{evalcore.BuiltinPrefix + "task_adherence", "support-agent-quality"}, + defaultEvaluators("support-agent-quality", false)) +} + +// A trace-backed eval has no target whose instructions a rubric is written +// from, so proposing one would plan a file nothing can generate. +func TestDefaultEvaluatorsSkipTheRubricForTraces(t *testing.T) { + assert.Equal(t, + []string{evalcore.BuiltinPrefix + "task_adherence"}, + defaultEvaluators("support-agent-quality", true)) +} + +// The prompt offers what is knowable without a service call -- init makes none +// -- which is the pair it proposes plus whatever the catalog already declares. +func TestEvaluatorChoicesOfferTheCatalogToo(t *testing.T) { + cfg := &project.EvalConfig{Evaluators: []project.EvaluatorDecl{ + {Name: "support-agent-quality"}, // already proposed, must not double up + {Name: "tone-check"}, + }} + + got := evaluatorChoices(cfg, "support-agent-quality", false) + + assert.Equal(t, []string{ + evalcore.BuiltinPrefix + "task_adherence", + "support-agent-quality", + "tone-check", + }, got) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go new file mode 100644 index 00000000000..330cc908f09 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "sort" + "strings" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// judgeModelEnvKey is the key `azd ai agent init` writes the bound deployment +// to, and the only place it appears when the project was created beforehand. +const judgeModelEnvKey = "AZURE_AI_MODEL_DEPLOYMENT_NAME" + +// modelDeployments names every model deployment the project declares, sorted so +// a prompt and an error list the same way twice. +// +// They live under the Foundry project service's deployments:, which is where +// the sibling extensions put them, so reading them is a file read rather than a +// service call. An entry given as a $ref is skipped rather than followed: +// resolving includes is the project extension's job, and a scaffold that cannot +// see a deployment falls back to naming --judge-model, which is a worse message +// but never a wrong one. +func modelDeployments(proj *azdext.ProjectConfig) []string { + seen := map[string]bool{} + var names []string + + for _, svc := range proj.GetServices() { + if svc.GetHost() != aiProjectHost { + continue + } + props := svc.GetAdditionalProperties() + if props == nil || len(props.GetFields()) == 0 { + props = svc.GetConfig() + } + if props == nil { + continue + } + declared, ok := props.AsMap()["deployments"].([]any) + if !ok { + continue + } + for _, entry := range declared { + fields, ok := entry.(map[string]any) + if !ok { + continue + } + name, ok := fields["name"].(string) + if !ok || name == "" || seen[name] { + continue + } + seen[name] = true + names = append(names, name) + } + } + + sort.Strings(names) + return names +} + +// resolveJudgeModel settles the deployment the graders judge with. +// +// The judging built-ins declare the deployment as required, so an eval written +// without one is rejected by the service long after the command that wrote it. +// That is why coming back empty is a failure here rather than something left +// for later: `init` would otherwise exit 0 having written a configuration that +// cannot be deployed. +func resolveJudgeModel(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, error) { + if model := detectModelDeployment(proj); model != "" { + return model, nil + } + + deployments := modelDeployments(proj) + switch len(deployments) { + case 0: + // Binding to an existing Foundry project writes `deployments: []` into + // azure.yaml, so the deployment `azd ai agent init` chose survives only + // in the azd environment. Reading it there is the difference between a + // configured project working and erroring. + if model := modelDeploymentFromAzdEnv(commandContext(cmd)); model != "" { + return model, nil + } + return "", messages.JudgeModelRequired() + case 1: + return deployments[0], nil + } + + if noPrompt(cmd) { + return "", messages.AmbiguousJudgeModel(deployments) + } + return promptJudgeModel(cmd, deployments) +} + +// tracesConnected reports whether the azd environment records an Application +// Insights connection, which is what `generate --from` already defaults on. +// +// A local read, so `init` keeps its promise to make no service calls. Absence +// is ordinary: init runs outside an azd project too. +func tracesConnected(ctx context.Context) bool { + return azdEnvValue(ctx, appInsightsEnvKey) != "" +} + +// modelDeploymentFromAzdEnv reads the deployment `azd ai agent init` recorded +// in the active azd environment. Absence is ordinary: `init` runs outside an +// azd project too, and the caller falls back to naming --judge-model. +func modelDeploymentFromAzdEnv(ctx context.Context) string { + return azdEnvValue(ctx, judgeModelEnvKey) +} + +// azdEnvValue reads one key from the azd environment this invocation acts on, +// answering empty whenever there is no daemon, no environment, or no such key. +func azdEnvValue(ctx context.Context, key string) string { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "" + } + defer azdClient.Close() + + envName := azdEnvironmentName(ctx, azdClient) + if envName == "" { + return "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envName, + Key: key, + }) + if err != nil { + return "" + } + return val.GetValue() +} + +// validateEvaluatorRefs rejects references that cannot name an evaluator. +// +// This needs no service call, so it keeps `init`'s promise to make none. The +// alternative is a config that scaffolds cleanly and fails at `create`, naming +// a value the user passed to a different command. +func validateEvaluatorRefs(refs []string) error { + for _, ref := range refs { + if strings.TrimSpace(ref) == "" { + return messages.EvaluatorRefEmpty() + } + if strings.ContainsAny(ref, " \t") { + return messages.EvaluatorRefMalformed(ref) + } + } + return nil +} + +// promptJudgeModel asks which of the project's deployments to judge with. +func promptJudgeModel(cmd *cobra.Command, deployments []string) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + choices := make([]*azdext.SelectChoice, 0, len(deployments)) + for i := range deployments { + choices = append(choices, &azdext.SelectChoice{ + Label: deployments[i], Value: deployments[i], + }) + } + + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectJudgeModelPrompt(), + Choices: choices, + }, + }) + if err != nil { + return "", messages.SelectingJudgeModel(err) + } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue and would read as the first deployment rather than as no answer. + if resp == nil || resp.Value == nil { + return "", messages.AmbiguousJudgeModel(deployments) + } + index := int(resp.GetValue()) + if index < 0 || index >= len(deployments) { + return "", messages.AmbiguousJudgeModel(deployments) + } + return deployments[index], nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go new file mode 100644 index 00000000000..39d6612ac00 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// projectWithDeployments builds a project whose Foundry project service +// declares the given deployments, which is the shape azd hands the extension +// for a `deployments:` list in azure.yaml. +func projectWithDeployments(t *testing.T, names ...string) *azdext.ProjectConfig { + t.Helper() + declared := make([]any, 0, len(names)) + for _, n := range names { + declared = append(declared, map[string]any{ + "name": n, + "model": map[string]any{"name": n, "format": "OpenAI", "version": "1"}, + }) + } + proj := projectWith() + proj.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", + Host: aiProjectHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployments": declared, + }), + } + return proj +} + +// The deployments live under the Foundry project service, which is where the +// sibling extensions put them. Reading them is a file read, so `init` keeps +// making no service calls. +func TestModelDeployments_ReadFromTheProjectService(t *testing.T) { + assert.Empty(t, modelDeployments(projectWith("api", "web")), + "a project with no Foundry project service declares no deployments") + + assert.Equal(t, []string{"gpt-4.1-nano", "gpt-4o-mini"}, + modelDeployments(projectWithDeployments(t, "gpt-4o-mini", "gpt-4.1-nano")), + "sorted, so a prompt and an error list them the same way twice") +} + +// A single declared deployment is the one to judge with, so the common project +// needs no flag at all. +func TestResolveJudgeModel_DetectsTheOnlyDeployment(t *testing.T) { + model, err := resolveJudgeModel(newInitCommand(), + projectWithDeployments(t, "gpt-4.1-nano")) + + require.NoError(t, err) + assert.Equal(t, "gpt-4.1-nano", model) +} + +// The judging built-ins declare the deployment as required, so a project that +// declares none has to say which to use. Failing here is the point: `init` used +// to exit 0 having written a configuration the service later rejects. +func TestResolveJudgeModel_NoDeploymentsNamesTheFlag(t *testing.T) { + _, err := resolveJudgeModel(newInitCommand(), projectWith("api", "web")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--judge-model") +} + +// With several there is nothing to detect. Under --no-prompt the flag is the +// only way to say which, so the error names it and lists the candidates. +func TestResolveJudgeModel_AmbiguousUnderNoPrompt(t *testing.T) { + cmd := newInitCommand() + // --no-prompt is inherited from the root in the real tree, so the test has + // to supply it the way the root does. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := resolveJudgeModel(cmd, projectWithDeployments(t, "gpt-4.1-nano", "gpt-4o-mini")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--judge-model") + assert.Contains(t, err.Error(), "gpt-4.1-nano") + assert.Contains(t, err.Error(), "gpt-4o-mini") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go new file mode 100644 index 00000000000..0c9ae913601 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Now that init asks rather than defaults, the rubric arrives as an explicit +// choice instead of through the empty-evaluators branch. It still has to be +// generated: the configuration declares a file, and if nothing produces it, +// `create` fails looking for it -- which is finding 0b, reintroduced by the +// prompt if the chosen path did not set this. +func TestScaffoldGeneratesARubricThatWasChosenRatherThanDefaulted(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{ + evalcore.BuiltinPrefix + "task_adherence", + "support-agent-quality", + }, + }) + + assert.True(t, plan.generateRubric, + "a chosen rubric still has to be generated, or its file never exists") + + require.Len(t, cfg.Evaluators, 1, "only the rubric is a catalog entry; the builtin is not") + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name) +} + +// A built-in is resolved by the service and has no local file, so choosing only +// built-ins must not plan a generation. +func TestScaffoldGeneratesNoRubricForBuiltinsAlone(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{evalcore.BuiltinPrefix + "task_adherence"}, + }) + + assert.False(t, plan.generateRubric) + assert.Empty(t, cfg.Evaluators) +} + +// An evaluator the author already has on disk is not the rubric init offers, so +// it is declared without planning a generation over it. +func TestScaffoldDoesNotGenerateAnUnrelatedEvaluator(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{"tone-check"}, + }) + + assert.False(t, plan.generateRubric, + "only the rubric init offers to write is one it knows how to generate") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "tone-check", cfg.Evaluators[0].Name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go new file mode 100644 index 00000000000..9b03ec1d8aa --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "sort" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// agentServices names every agent the project declares, sorted so a prompt and +// an error list the same way twice. +func agentServices(proj *azdext.ProjectConfig) []string { + var names []string + for name, svc := range proj.GetServices() { + if svc.GetHost() == project.AgentHost { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +// resolveAgentTarget settles which agent the scaffold is written for. +// +// The spec makes --target default to the project's only agent, so requiring it +// would put a flag in front of the one command a developer runs first. With +// several agents there is nothing to detect, so it asks; under --no-prompt it +// names the flag rather than guessing which agent someone meant. +// +// It does not announce what it found: init prints that for every target, so +// that an explicit --target and a detected one read the same. +func resolveAgentTarget(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, error) { + agents := agentServices(proj) + switch len(agents) { + case 0: + return "", messages.NoAgentToEvaluate() + case 1: + return agents[0], nil + } + + if noPrompt(cmd) { + return "", messages.AmbiguousAgentTarget(agents) + } + return promptAgentTarget(cmd, agents) +} + +// promptAgentTarget asks which of the project's agents to evaluate. +func promptAgentTarget(cmd *cobra.Command, agents []string) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + choices := make([]*azdext.SelectChoice, 0, len(agents)) + for i := range agents { + choices = append(choices, &azdext.SelectChoice{Label: agents[i], Value: agents[i]}) + } + + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectAgentPrompt(), + Choices: choices, + }, + }) + if err != nil { + return "", messages.SelectingAgent(err) + } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue and would read as the first agent rather than as no answer. + if resp == nil || resp.Value == nil { + return "", messages.AmbiguousAgentTarget(agents) + } + index := int(resp.GetValue()) + if index < 0 || index >= len(agents) { + return "", messages.AmbiguousAgentTarget(agents) + } + return agents[index], nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go new file mode 100644 index 00000000000..9cef7d72df4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// projectWithHosts builds a project whose services carry hosts, which is what +// agent detection keys on. The sibling projectWith helper sets names only. +func projectWithHosts(services map[string]string) *azdext.ProjectConfig { + svcs := map[string]*azdext.ServiceConfig{} + for name, host := range services { + svcs[name] = &azdext.ServiceConfig{Name: name, Host: host} + } + return &azdext.ProjectConfig{Services: svcs} +} + +// The spec's first hero command is `azd ai eval init --source traces`, with no +// target. Requiring the flag put it in front of the one command a developer +// runs first. +func TestAgentServices_FindsTheOnlyAgent(t *testing.T) { + proj := projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + }) + + agents := agentServices(proj) + + require.Len(t, agents, 1) + assert.Equal(t, "support-agent", agents[0]) +} + +// Sorted, so a prompt and an error list them the same way twice. +func TestAgentServices_AreSorted(t *testing.T) { + proj := projectWithHosts(map[string]string{ + "zebra-agent": project.AgentHost, + "alpha-agent": project.AgentHost, + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + }) + + assert.Equal(t, []string{"alpha-agent", "support-agent", "zebra-agent"}, agentServices(proj)) +} + +func TestAgentServices_NoneWhenTheProjectHasNoAgent(t *testing.T) { + proj := projectWithHosts(map[string]string{"ai-project": "azure.ai.project"}) + + assert.Empty(t, agentServices(proj)) +} + +// A project with no agent cannot be scaffolded, and the error has to say that +// rather than name a flag the developer has nothing to put in. +func TestResolveAgentTarget_NoAgent(t *testing.T) { + cmd := newInitCommand() + + _, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no agent service") +} + +// With several agents there is nothing to detect. Under --no-prompt the flag is +// the only way to say which, so the error names it and lists the candidates. +func TestResolveAgentTarget_AmbiguousUnderNoPrompt(t *testing.T) { + cmd := newInitCommand() + // --no-prompt is inherited from the root in the real tree, so the test has + // to supply it the way the root does. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "one-agent": project.AgentHost, + "two-agent": project.AgentHost, + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--target") + assert.Contains(t, err.Error(), "one-agent") + assert.Contains(t, err.Error(), "two-agent") +} + +func TestResolveAgentTarget_DetectsTheSoleAgent(t *testing.T) { + cmd := newInitCommand() + + target, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + })) + + require.NoError(t, err) + assert.Equal(t, "support-agent", target) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go new file mode 100644 index 00000000000..7eca43c1e76 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -0,0 +1,452 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// projectCanProvision decides which deploy command `init` names, so it has to +// agree with what azd actually does rather than with what the layout suggests. +// azd infers the provider from the files in the infra directory; an empty or +// missing directory leaves it unspecified, falls back to Bicep, and fails on +// the absent infra/main.bicep. +func TestProjectCanProvision(t *testing.T) { + write := func(t *testing.T, dir string, names ...string) string { + t.Helper() + root := t.TempDir() + if dir != "" { + require.NoError(t, os.MkdirAll(filepath.Join(root, dir), 0o750)) + } + for _, n := range names { + require.NoError(t, + os.WriteFile(filepath.Join(root, dir, n), []byte("// x"), 0o600)) + } + return root + } + + t.Run("no infra directory", func(t *testing.T) { + root := write(t, "") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root}), + "this is the eval-only project, where `azd up` exits 1") + }) + + t.Run("infra directory with bicep", func(t *testing.T) { + root := write(t, "infra", "main.bicep") + require.True(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + t.Run("infra directory with terraform", func(t *testing.T) { + root := write(t, "infra", "main.tf") + require.True(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + // An empty directory is the case a plain os.Stat would get wrong: the + // directory exists, and azd still has nothing to compile. + t.Run("empty infra directory", func(t *testing.T) { + root := write(t, "infra") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + // Nothing recurses: azd reads one directory and skips subdirectories. + t.Run("bicep only in a subdirectory", func(t *testing.T) { + root := write(t, filepath.Join("infra", "modules"), "db.bicep") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + t.Run("project names its own infra directory", func(t *testing.T) { + root := write(t, "deploy", "main.bicep") + require.True(t, projectCanProvision(&azdext.ProjectConfig{ + Path: root, + Infra: &azdext.InfraOptions{Path: "deploy"}, + }), "the declared path is read, not the default one") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root}), + "and the default is empty here") + }) + + t.Run("no project", func(t *testing.T) { + require.False(t, projectCanProvision(nil), + "an unreadable project cannot be claimed to provision") + }) +} + +// scaffoldFor runs planScaffold against a fresh configuration, which is what +// `init` does on a project that has never been initialized. +func scaffoldFor(t *testing.T, in scaffoldInput) (scaffold, *project.EvalConfig) { + t.Helper() + if in.cfg == nil { + in.cfg = &project.EvalConfig{} + } + if in.evalDir == "" { + in.evalDir = project.DefaultEvalDir + } + if in.rubricName == "" { + in.rubricName = in.target + "-quality" + } + return planScaffold(in), in.cfg +} + +// The scaffold must round-trip and validate, otherwise `azd up` fails on a +// config the tool itself produced. +func TestScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + evalDir: dir, + }) + + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") + + eval, err := loaded.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) + require.Equal(t, project.EvaluationLevelTurn, eval.EvaluationLevel) +} + +// Re-running init appends rather than replacing, so one file ends up holding +// every eval for the target. +func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "first", target: "support-agent", judgeModel: "m", evalDir: dir, + }) + _, cfg = scaffoldFor(t, scaffoldInput{ + evalName: "second", target: "support-agent", judgeModel: "m", evalDir: dir, cfg: cfg, + }) + + require.Equal(t, []string{"first", "second"}, cfg.EvalNames()) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A trace-backed eval invokes nothing, so agent_name filters instead of +// targeting, and a scaffolded cap keeps the first run bounded rather than +// taking the service's default of 1000. +func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + maxTraces: project.DefaultScaffoldMaxTraces, + }) + + require.Nil(t, plan.eval.Target) + require.NotNil(t, plan.eval.Source) + require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) + require.Equal(t, "support-agent", plan.eval.Source.AgentName) + require.Equal(t, 20, plan.eval.Source.MaxTraces) +} + +// Omitting the cap leaves the key out, which is how the service default is +// taken — writing a zero would send one. +func TestScaffold_TraceCapIsOmittedWhenZero(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "t", target: "a", source: initSourceTraces, + }) + require.Zero(t, plan.eval.Source.MaxTraces) + + body, err := yaml.Marshal(plan.eval) + require.NoError(t, err) + require.NotContains(t, string(body), "max_traces") +} + +// The default set is a built-in plus a generated rubric: the built-in alone +// would be generic, and the rubric is what makes the baseline about this agent. +func TestScaffold_DefaultEvaluators(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "gpt-5.6-luna", + }) + + require.Equal(t, + []string{"builtin.task_adherence", "support-agent-quality"}, + plan.evaluatorNames()) + + // Every evaluator carries the judge deployment, because the judging + // built-ins declare it and an eval that leaves it off is rejected. + for _, ref := range plan.eval.Evaluators { + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["model"], + "%s must name a judge deployment", ref.Evaluator) + } +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation. +func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) + + require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) + require.False(t, plan.generateRubric, "no rubric is generated when evaluators are given") +} + +// `init` closes by naming what to run next, and only what has something to do. +// Pointing a caller who supplied their own artifacts at a generation command +// would submit a billed job for something they already have. +func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { + t.Run("nothing supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", + }) + // One command produces both, so there is one step, not two. + require.Equal(t, + []string{"azd ai eval generate --target support-agent --generation-model m"}, + plan.nextSteps("azd ai eval create")) + }) + + t.Run("dataset supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", + }) + require.Equal(t, + []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality " + + "--target support-agent --generation-model m"}, + plan.nextSteps("azd ai eval create")) + }) + + t.Run("everything supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) + // Verified against azd 1.30.0. `azd up` on a project with no infra/ + // exits 1 compiling a missing infra/main.bicep, and `azd deploy` exits + // 1 with "infrastructure has not been provisioned" in an environment + // that never provisioned one. `azd ai eval create` needs neither. + require.Equal(t, []string{"azd ai eval create", "azd ai eval run start"}, + plan.nextSteps("azd ai eval create"), + "the deploy step is the one the project can actually run") + require.Equal(t, []string{"azd up", "azd ai eval run start"}, + plan.nextSteps("azd up"), + "where the project does provision, one command covers both") + }) +} + +// Which command deploys is decided in one place, so every message that names +// one agrees. The detection itself is covered by TestProjectCanProvision. +func TestDeployCommandName(t *testing.T) { + root := t.TempDir() + require.Equal(t, "azd ai eval create", deployCommandName(&azdext.ProjectConfig{Path: root}), + "nothing to provision, so neither `azd up` nor `azd deploy` would run here") + + require.NoError(t, os.MkdirAll(filepath.Join(root, "infra"), 0o750)) + require.NoError(t, + os.WriteFile(filepath.Join(root, "infra", "main.bicep"), []byte("// x"), 0o600)) + require.Equal(t, "azd up", deployCommandName(&azdext.ProjectConfig{Path: root})) + + require.Equal(t, "azd ai eval create", deployCommandName(nil), + "a project we cannot read is not one we can claim provisions") +} + +// The literals above are only as good as the surface they name. This resolves +// every step against the real command tree, so a step naming a command that has +// been renamed or removed fails here rather than in a user's terminal — which +// is how `azd ai eval dataset generate` survived being deleted. +func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { + inputs := []scaffoldInput{ + {evalName: "smoke", target: "support-agent", judgeModel: "m"}, + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m"}, + // Reaches the deploy branch, so the command it names is resolved too. + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, judgeModel: "m"}, + } + + for _, in := range inputs { + plan, _ := scaffoldFor(t, in) + for _, step := range plan.nextSteps("azd ai eval create") { + // Steps that drive azd itself -- `azd up`, `azd deploy` -- are not + // this extension's commands and resolve against a different tree. + if !strings.HasPrefix(step, "azd ai eval ") { + continue + } + words := strings.Fields(strings.TrimPrefix(step, "azd ai eval ")) + if len(words) == 0 { + continue + } + // Stop at the first flag: what follows is arguments, not commands. + var path []string + for _, w := range words { + if strings.HasPrefix(w, "-") { + break + } + path = append(path, w) + } + + cmd, rest, err := NewRootCommand().Find(path) + require.NoErrorf(t, err, "%q names no command", step) + require.Emptyf(t, rest, "%q left %v unresolved, so it is not a command", step, rest) + require.Equalf(t, path[len(path)-1], strings.Fields(cmd.Use)[0], + "%q resolved to %q, not the command it names", step, cmd.Use) + } + } +} + +// The Next: line is what a new user runs immediately after init, and it used to +// fail twice before it worked: `generate` requires --target and +// --generation-model, detects neither, and reports them one per invocation. +// Both values were on screen when init printed the hint. +func TestScaffold_NextStepsCarryWhatGenerateRequires(t *testing.T) { + for _, in := range []scaffoldInput{ + {evalName: "smoke", target: "support-agent", judgeModel: "gpt-4o-mini"}, + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "gpt-4o-mini"}, + } { + plan, _ := scaffoldFor(t, in) + seen := 0 + for _, step := range plan.nextSteps("azd ai eval create") { + if !strings.HasPrefix(step, "azd ai eval generate") { + continue + } + seen++ + require.Containsf(t, step, "--target support-agent", + "%q omits the target init had just detected", step) + require.Containsf(t, step, "--generation-model gpt-4o-mini", + "%q omits the model init had just resolved", step) + } + require.NotZerof(t, seen, + "no generate step was produced, so this asserted nothing: %+v", in) + } +} + +// Built-ins are referenced but never declared, so the scaffold must not give +// one a catalog entry to publish. +func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { + dir := t.TempDir() + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence", "my-custom"}, + judgeModel: "m", + evalDir: dir, + }) + + require.Len(t, plan.eval.Evaluators, 2) + require.True(t, plan.eval.Evaluators[0].IsBuiltin()) + require.False(t, plan.eval.Evaluators[1].IsBuiltin()) + + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator is declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + require.Len(t, cfg.CustomEvaluators(), 1, + "only the custom evaluator is this config's to publish") + + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A bare name means an already-registered dataset; a path means a local file. +// Either way the dataset was supplied, so nothing is scheduled to generate it — +// only a missing --dataset produces a generation step. +func TestScaffold_DatasetReferenceForms(t *testing.T) { + t.Run("local path becomes a source", func(t *testing.T) { + // --dataset is relative to the working directory, but source: is + // resolved relative to the eval config, so it has to be rebased. + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "./tests/golden.jsonl", evalDir: "evals", + }) + decl, ok := cfg.DatasetDeclaration("golden") + require.True(t, ok) + require.Equal(t, "../tests/golden.jsonl", decl.Source, + "a dataset outside the eval dir must be reached with ..") + require.Equal(t, "golden", plan.eval.Dataset) + require.False(t, plan.generateDataset, + "a supplied dataset must not be scheduled for generation") + }) + + t.Run("bare name references a registered dataset", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "prod-sample", + }) + decl, ok := cfg.DatasetDeclaration("prod-sample") + require.True(t, ok) + require.Empty(t, decl.Source, "a registered dataset must not get a local source") + require.Equal(t, "prod-sample", plan.eval.Dataset) + require.False(t, plan.generateDataset) + }) + + t.Run("no dataset flag scaffolds a local path and a generation step", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", + }) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset, + "the dataset is named after the eval") + decl, ok := cfg.DatasetDeclaration("support-agent-smoke") + require.True(t, ok) + require.Contains(t, decl.Source, "support-agent-smoke.jsonl") + require.True(t, plan.generateDataset) + }) +} + +func TestLooksLikeLocalDataset(t *testing.T) { + require.True(t, looksLikeLocalDataset("./data/golden.jsonl")) + require.True(t, looksLikeLocalDataset("golden.jsonl")) + require.True(t, looksLikeLocalDataset(`data\golden.jsonl`)) + require.False(t, looksLikeLocalDataset("prod-sample")) +} + +// Paths are used verbatim relative to the working directory; the doubling bug +// in the agent-scoped command must not reappear. +func TestSaveEvalConfig_UsesPathVerbatim(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "evals") + + require.NoError(t, project.SaveEvalConfig(nested, &project.EvalConfig{})) + _, err := os.Stat(project.EvalConfigPath(nested)) + require.NoError(t, err, "the file must land exactly at the requested path") + + doubled := filepath.Join(dir, "evals", "evals") + _, err = os.Stat(doubled) + require.Error(t, err, "the path must not be re-rooted under itself") +} + +// normalizeRubricBody accepts a bare definition or a full document. +func TestNormalizeRubricBody(t *testing.T) { + t.Run("bare definition is wrapped", func(t *testing.T) { + body, err := normalizeRubricBody("quality", + []byte(`{"type":"rubric","dimensions":[{"id":"q","weight":10}]}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"quality"`) + require.Contains(t, string(body), `"definition"`) + }) + + t.Run("full document keeps its definition and takes the flag name", func(t *testing.T) { + body, err := normalizeRubricBody("renamed", + []byte(`{"name":"old","definition":{"type":"rubric","dimensions":[]}}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"renamed"`) + }) + + t.Run("rejects a document with neither", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"unrelated":true}`)) + require.ErrorContains(t, err, "dimensions") + }) + + t.Run("rejects invalid JSON", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`not json`)) + require.ErrorContains(t, err, "not valid JSON") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go new file mode 100644 index 00000000000..03745d01e75 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func projectWith(names ...string) *azdext.ProjectConfig { + proj := &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{}} + for _, n := range names { + proj.Services[n] = &azdext.ServiceConfig{Name: n} + } + return proj +} + +// The eval service is ordered after everything it reads, but only names +// services the project actually declares. Naming one it does not have is a +// broken reference, and an eval config can sit in a repo that reaches an +// existing Foundry project by endpoint and an agent deployed elsewhere. +func TestEvalServiceUses_OnlyWhatTheProjectDeclares(t *testing.T) { + assert.Nil(t, evalServiceUses(projectWith("api", "web"), "support-agent"), + "neither the project service nor the agent is declared, so there is nothing to order after") + + withProject := projectWith("api", "support-agent") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project", "support-agent"}, + evalServiceUses(withProject, "support-agent"), + "the eval runs after the project it evaluates against and the agent it evaluates") + + assert.Equal(t, []string{"support-agent"}, + evalServiceUses(projectWith("support-agent"), "support-agent"), + "an agent alone is still worth ordering after") +} + +// `init` detects the judge deployment from the project, because it makes no +// service calls and this is the only place it can read one. +func TestDetectModelDeployment(t *testing.T) { + assert.Empty(t, detectModelDeployment(projectWith("api", "web"))) + + proj := projectWith("api") + proj.Services["chat"] = &azdext.ServiceConfig{Name: "chat", Host: aiModelHost} + assert.Equal(t, "chat", detectModelDeployment(proj), + "the service name is the deployment name when nothing more specific is declared") + + named := projectWith() + named.Services["chat"] = &azdext.ServiceConfig{ + Name: "chat", + Host: aiModelHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployment": "gpt-5.6-luna", + }), + } + assert.Equal(t, "gpt-5.6-luna", detectModelDeployment(named)) +} + +func mustStruct(t *testing.T, m map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(m) + require.NoError(t, err) + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go new file mode 100644 index 00000000000..52279b70b9b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A useful generation instruction is often longer than fits on a command +// line, so it can come from a file instead. +func TestResolveInstructionReadsFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(path, + []byte(" A customer support agent answering billing questions.\n\n"), 0o600)) + + got, err := resolveInstruction("", path) + require.NoError(t, err) + require.Equal(t, "A customer support agent answering billing questions.", got, + "surrounding whitespace should be trimmed") +} + +func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + + got, err = resolveInstruction("", "") + require.NoError(t, err) + require.Empty(t, got) +} + +// An unreadable or empty file is reported rather than silently generating from +// no instruction at all. +func TestResolveInstructionRejectsUnusableFile(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "agent-instruction-file") + + empty := filepath.Join(t.TempDir(), "empty.md") + require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) + _, err = resolveInstruction("", empty) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go new file mode 100644 index 00000000000..5f299788cb3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Generation runs as two independent long-running resources — one for datasets, +// one for evaluators — sharing no collection. A job group therefore nests under +// the resource that produced it: a top-level `job show ` would have to guess +// the endpoint from an id prefix that is not a documented contract. + +const ( + jobKindDataset = "dataset" + jobKindEvaluator = "evaluator" +) + +// jobKind binds a group to one generation resource, so every command under it +// calls one endpoint rather than trying both and reporting whichever answered. +type jobKind struct { + name string + list func(context.Context, *evalContext) ([]eval_api.GenerationJob, error) + get func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + cancel func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + remove func(context.Context, *evalContext, string) error +} + +// Data generation is the one collection on its own API version, so the job +// commands have to ask for it the same way generate does. Evaluator generation +// is on the project endpoint version, which is why only these four differ. +var datasetJobs = jobKind{ + name: jobKindDataset, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListDataGenerationJobs(ctx, DataGenerationAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetDataGenerationJob(ctx, id, DataGenerationAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelDataGenerationJob(ctx, id, DataGenerationAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteDataGenerationJob(ctx, id, DataGenerationAPIVersion) + }, +} + +var evaluatorJobs = jobKind{ + name: jobKindEvaluator, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +// jobSelector binds a command to one of the two generation collections. +// +// Required here, unlike on `generate`: an id alone does not say which +// collection to call, and the two share an id shape, so guessing would mean +// trying both and reporting whichever answered. +type jobSelector struct { + dataset bool + evaluator bool +} + +func (s *jobSelector) bind(cmd *cobra.Command) { + // "Required." leads, because the same two flag names are optional filters + // one command over on `generate`, and the help is the only thing that says + // which meaning applies here. + cmd.Flags().BoolVar(&s.dataset, "dataset", false, + "Required (or --evaluator). Act on dataset generation jobs.") + cmd.Flags().BoolVar(&s.evaluator, "evaluator", false, + "Required (or --dataset). Act on evaluator generation jobs.") + cmd.MarkFlagsMutuallyExclusive("dataset", "evaluator") + cmd.MarkFlagsOneRequired("dataset", "evaluator") +} + +// kind resolves the selector. Total rather than defaulting: cobra enforces +// that one flag is set, and if that enforcement is ever dropped a silent +// default would query the wrong collection and report "not found". +func (s *jobSelector) kind() (jobKind, error) { + switch { + case s.dataset && !s.evaluator: + return datasetJobs, nil + case s.evaluator && !s.dataset: + return evaluatorJobs, nil + default: + return jobKind{}, messages.JobKindRequired() + } +} + +func newJobCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "job", + Short: "Inspect, cancel and delete generation jobs.", + Long: "Inspect, cancel and delete generation jobs.\n\n" + + "This is the resume path for `generate`: a job started with --no-wait, " + + "or one whose client was interrupted, is reattached to here rather than " + + "restarted.\n\n" + + "Pass --dataset or --evaluator to say which generation to act on. " + + "The two are separate service collections, so it is required.", + } + cmd.AddCommand( + newJobListCommand(), + newJobShowCommand(), + newJobCancelCommand(), + newJobDeleteCommand(), + ) + return cmd +} + +func newJobListCommand() *cobra.Command { + var endpointFlg string + sel := &jobSelector{} + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's generation jobs.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + kind, err := sel.kind() + if err != nil { + return err + } + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + jobs, err := kind.list(ctx, ec) + if err != nil { + return messages.ListingJobs(kind.name, err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), jobs) + } + if len(jobs) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoJobs(kind.name)) + return nil + } + table := make([][]string, 0, len(jobs)) + for _, j := range jobs { + table = append(table, []string{j.ID, j.Status}) + } + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) + }, + } + + sel.bind(cmd) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobShowCommand() *cobra.Command { + var endpointFlg string + sel := &jobSelector{} + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + kind, err := sel.kind() + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + job, err := kind.get(ctx, ec, jobID) + if err != nil { + return jobLookupError("reading", kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), job) + } + fmt.Fprint(cmd.OutOrStdout(), messages.JobLine(job.ID, job.Status)) + if job.Error != nil && job.Error.Message != "" { + fmt.Fprint(cmd.OutOrStdout(), messages.JobErrorLine(job.Error.Message)) + } + return nil + }, + } + + sel.bind(cmd) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobCancelCommand() *cobra.Command { + var endpointFlg string + sel := &jobSelector{} + + cmd := &cobra.Command{ + Use: "cancel ", + Short: "Cancel an in-flight generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + kind, err := sel.kind() + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + canceled, err := kind.cancel(ctx, ec, jobID) + if err != nil { + return jobLookupError("cancelling", kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + fmt.Fprint(cmd.OutOrStdout(), + messages.JobCancelled(kind.name, jobID, canceled.Status)) + return nil + }, + } + + sel.bind(cmd) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobDeleteCommand() *cobra.Command { + var endpointFlg string + sel := &jobSelector{} + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a generation job record.", + Long: "Delete a generation job record.\n\n" + + "The artifact the job produced is already registered as its own version " + + "and is not affected.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + kind, err := sel.kind() + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := kind.remove(ctx, ec, jobID); err != nil { + return jobLookupError("deleting", kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": jobID, "kind": kind.name, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.JobDeleted(kind.name, jobID)) + return nil + }, + } + + sel.bind(cmd) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// jobLookupError names the sibling group, because the two job types share an id +// shape and reaching for the wrong one is the likely mistake. +// +// action is what the caller was doing, so a failed delete does not report that +// a read failed. +func jobLookupError(action string, kind jobKind, jobID string, err error) error { + if eval_api.IsNotFound(err) { + other := jobKindEvaluator + if kind.name == jobKindEvaluator { + other = jobKindDataset + } + return messages.JobNotFound(kind.name, jobID, other) + } + return messages.JobActionFailed(action, kind.name, jobID, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go new file mode 100644 index 00000000000..7cf4fc48788 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeJSONL(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "d.jsonl") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// The service accepts whatever bytes it is given, so a malformed row becomes a +// published version with an eval bound to it, and only fails much later +// on a row nobody has looked at. A live deploy published `{not json at all}` +// as version 1.0 before this existed. +func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"fine\"}\n{not json at all}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "one JSON object") +} + +func TestValidateJSONL_AcceptsWellFormedRows(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n{\"query\":\"b\"}\n"))) +} + +// Trailing and interior blank lines are formatting, not rows. +func TestValidateJSONL_IgnoresBlankLines(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n\n{\"query\":\"b\"}\n\n"))) +} + +// A file with nothing in it publishes a version that can never score anything. +func TestValidateJSONL_RejectsAFileWithNoRows(t *testing.T) { + err := validateJSONL(writeJSONL(t, "\n\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} + +// A JSON array is the shape people reach for when they mean JSONL. +func TestValidateJSONL_RejectsAJSONArray(t *testing.T) { + err := validateJSONL(writeJSONL(t, "[{\"query\":\"a\"},{\"query\":\"b\"}]\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 1") +} + +// An empty object parses but evaluates to nothing. +func TestValidateJSONL_RejectsAnEmptyObject(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"a\"}\n{}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "empty object") +} + +// A conversation-level row holds a whole transcript and runs past bufio's +// default 64KB line limit, which would otherwise be reported as invalid JSON. +func TestValidateJSONL_AcceptsAVeryLongRow(t *testing.T) { + long := make([]byte, 200*1024) + for i := range long { + long[i] = 'x' + } + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\""+string(long)+"\"}\n"))) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go new file mode 100644 index 00000000000..72e17abae81 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run reattached by id repeats the data source the last one sent, so an eval +// whose last run predates the preview shape would keep sending the old one for +// good. The old shape carried no agent version, which is the whole reason to +// move off it. +func TestPinReusedTraceWindow_MovesTheLegacyShapeOn(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + MaxTraces: 500, + }) + + require.NotNil(t, ds.TraceSource) + assert.Equal(t, eval_api.EvalRunDataSourceTypeTracePreview, ds.Type) + assert.Equal(t, "agent_filter", ds.TraceSource.Type) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + assert.Equal(t, 500, ds.TraceSource.MaxTraces) + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime, 60) + // Nothing was pinned before, so nothing is pinned now: the upgrade must not + // invent a version the previous runs were never graded against. + assert.Empty(t, ds.TraceSource.AgentVersion) +} + +// A window with a start and no end means "up to now", so replaying it a week +// later grades a week more than the run it was copied from, and the run after +// that more again. Both ends are written down, whatever shape the window +// arrived in, so the span cannot grow with each reattach. +func TestPinReusedTraceWindow_ClosesAWindowSoItCannotWiden(t *testing.T) { + cases := []struct { + name string + ds *eval_api.EvalRunDataSource + }{ + { + "a legacy source", + &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + }, + }, + { + // The shape this extension writes today. Pinning only the legacy + // one left the current one growing in exactly the way the legacy + // handling exists to prevent. + "a source this extension wrote", + &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", + AgentName: "support-agent", + StartTime: time.Now().Add(-24 * time.Hour).Unix(), + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ds := pinReusedTraceWindow(tc.ds) + + require.NotNil(t, ds.TraceSource) + assert.InDelta(t, time.Now().Unix(), ds.TraceSource.EndTime, 60) + assert.InDelta(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime, 60) + + // An already-closed window is repeated rather than re-pinned, so + // the span cannot creep run after run. Asserted by identity: two + // calls a microsecond apart write the same second, so comparing + // the values would pass with the guard deleted. + assert.Same(t, ds, pinReusedTraceWindow(ds)) + }) + } +} + +// The old shape had no start bound, so a run that set no lookback was graded +// over whatever the service chose. Carrying it forward with no start would +// widen it to all of history instead. +func TestPinReusedTraceWindow_KeepsTheWindowALegacyRunRanUnder(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + }) + + require.NotNil(t, ds.TraceSource) + assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) +} + +// The recorded values come from an older build, from before the bounds existed. +// A lookback past the bound reaches back further than a window may cover, and a +// negative cap is no cap at all: left as zero it is dropped from the request, +// which means the service's own default of a thousand traces -- a bigger and +// costlier run than the one being repeated. +func TestPinReusedTraceWindow_ClampsWhatAnOlderBuildRecorded(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: project.MaxLookbackHours + 1, + MaxTraces: -5, + }) + + require.NotNil(t, ds.TraceSource) + assert.Greater(t, ds.TraceSource.EndTime, ds.TraceSource.StartTime) + assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) + assert.Equal(t, project.DefaultScaffoldMaxTraces, ds.TraceSource.MaxTraces, + "a bounded cap, rather than none at all") +} + +// An end bound anchors the window it closes, rather than being read alongside a +// start measured from now: a run that ended a month ago covered the week before +// that, not the week before today. +func TestPinReusedTraceWindow_MeasuresBackFromTheEnd(t *testing.T) { + end := time.Date(2026, 8, 2, 0, 0, 0, 0, time.UTC) + + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + EndTime: end.Unix(), + }) + + require.NotNil(t, ds.TraceSource) + assert.Equal(t, end.Unix(), ds.TraceSource.EndTime) + assert.Equal(t, end.Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime) +} + +// Anything that is not an open trace window is repeated exactly, so a source +// this extension has never heard of is not quietly replaced with one it made up. +func TestPinReusedTraceWindow_LeavesEverythingElseAlone(t *testing.T) { + assert.Nil(t, pinReusedTraceWindow(nil)) + + closed := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", AgentName: "a", StartTime: 7, EndTime: 8, + }, + } + assert.Same(t, closed, pinReusedTraceWindow(closed)) + + jsonl := &eval_api.EvalRunDataSource{Type: eval_api.EvalRunDataSourceTypeJSONL} + assert.Same(t, jsonl, pinReusedTraceWindow(jsonl)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go new file mode 100644 index 00000000000..9cef816cec2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// newListenCommand registers the service-target provider with azd. It is hidden +// and invoked by azd itself, not by users. +func newListenCommand() *cobra.Command { + return azdext.NewListenCommand(configureExtensionHost) +} + +// configureExtensionHost wires the azure.ai.eval service target so `azd up` and +// `azd deploy` reach this extension. The provider name must match the manifest. +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + + host.WithServiceTarget(project.EvalHost, func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider( + azdClient, + func(ctx context.Context) (project.Reconciler, error) { + return newEvalReconciler(ctx) + }, + ) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go new file mode 100644 index 00000000000..b726e669783 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "testing" + + "github.com/fatih/color" +) + +// TestMain pins colour off for the whole package. +// +// fatih/color decides once, at init, from whether the process's stdout is a +// terminal -- not from the writer a renderer was handed. `go test` pipes +// stdout, so a test asserting on plain text passes under `go test` and fails +// when the compiled test binary is run from a terminal. Pinning it here makes +// the expected output the same either way, rather than leaving every assertion +// on a rendered line to depend on how the suite was started. +func TestMain(m *testing.M) { + color.NoColor = true + os.Exit(m.Run()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go new file mode 100644 index 00000000000..faf104043b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// extensionManifest is the subset of extension.yaml this test asserts on. +type extensionManifest struct { + ID string `yaml:"id"` + Version string `yaml:"version"` + Capabilities []string `yaml:"capabilities"` + Providers []struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + } `yaml:"providers"` +} + +func loadManifest(t *testing.T) extensionManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "extension.yaml")) + require.NoError(t, err, "reading extension.yaml") + + var manifest extensionManifest + require.NoError(t, yaml.Unmarshal(raw, &manifest)) + return manifest +} + +// A declared capability azd cannot reach is worse than an undeclared one: azd +// invokes `metadata` to discover the command tree, and it was declared without +// the command being registered, so discovery failed with "unknown command". +func TestDeclaredCapabilitiesAreImplemented(t *testing.T) { + manifest := loadManifest(t) + root := NewRootCommand() + + hasCommand := func(name string) bool { + for _, sub := range root.Commands() { + if sub.Name() == name { + return true + } + } + return false + } + + for _, capability := range manifest.Capabilities { + switch capability { + case "metadata": + require.True(t, hasCommand("metadata"), + "the metadata capability requires a metadata command") + case "service-target-provider": + require.True(t, hasCommand("listen"), + "a service-target provider is registered through the listen command") + require.NotEmpty(t, manifest.Providers, + "the manifest must name the provider it registers") + case "custom-commands": + require.NotEmpty(t, root.Commands()) + case "lifecycle-events": + // The SDK only starts the event manager when handlers are + // registered, so declaring this without any is an unused + // permission. Nothing here registers handlers today. + t.Fatalf("lifecycle-events is declared but no event handlers are registered") + } + } +} + +// The provider name in the manifest is what azd matches a service's `host` +// against, so a mismatch silently means the provider is never invoked. +func TestManifestProviderMatchesHostConstant(t *testing.T) { + manifest := loadManifest(t) + require.NotEmpty(t, manifest.Providers) + + names := make([]string, 0, len(manifest.Providers)) + for _, p := range manifest.Providers { + names = append(names, p.Name) + } + require.Contains(t, names, "azure.ai.eval", + "the manifest must declare the host the provider registers for") +} + +// extension.yaml carries a note asking that version.txt be kept in sync. The +// build stamps the binary from version.txt while the registry reads +// extension.yaml, so a drift ships a binary that misreports its own version. +func TestManifestVersionMatchesVersionFile(t *testing.T) { + manifest := loadManifest(t) + + raw, err := os.ReadFile(filepath.Join("..", "..", "version.txt")) + require.NoError(t, err, "reading version.txt") + + require.Equal(t, + strings.TrimSpace(string(raw)), + strings.TrimSpace(manifest.Version), + "version.txt and extension.yaml must agree") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go new file mode 100644 index 00000000000..c2448ececee --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordedUpdate is the body of an update the reconciler pushed, or nil when it +// pushed nothing. +type recordedUpdate struct { + body *eval_api.UpdateOpenAIEvalRequest +} + +// reconcilerHoldingEval builds a reconciler whose service holds this eval, and +// records any update pushed to it. +func reconcilerHoldingEval( + t *testing.T, + held eval_api.OpenAIEval, +) (*evalReconciler, *recordedUpdate) { + t.Helper() + seen := &recordedUpdate{} + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. + if r.Method == http.MethodPost { + raw, err := io.ReadAll(r.Body) + assert.NoError(t, err) + var body eval_api.UpdateOpenAIEvalRequest + assert.NoError(t, json.Unmarshal(raw, &body)) + seen.body = &body + } + w.Header().Set("Content-Type", "application/json") + assert.NoError(t, json.NewEncoder(w).Encode(held)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalReconciler{ec: &evalContext{ + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + }}, seen +} + +// A description is excluded from the fingerprint so that editing it does not +// fork the run history. Excluding it from the digest is not the same as +// ignoring it: the edit still has to reach the service, and this reuse path is +// the only place it can. +func TestPushMutableSendsAnEditedDescription(t *testing.T) { + r, seen := reconcilerHoldingEval(t, eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "old wording"}, + }) + + r.pushMutable(context.Background(), "eval-1", project.Eval{ + Name: "support-gate", + Description: "new wording", + }, &eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "old wording"}, + }) + + require.NotNil(t, seen.body, "an edited description must be pushed") + assert.Equal(t, "new wording", seen.body.Metadata[metaDescription]) + assert.Equal(t, "support-gate", seen.body.Name, "the name rides along unchanged") +} + +// A rename is applied in place rather than forking the history. +func TestPushMutableSendsARename(t *testing.T) { + held := eval_api.OpenAIEval{ID: "eval-1", Name: "old-name"} + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", + project.Eval{Name: "new-name"}, &held) + + require.NotNil(t, seen.body) + assert.Equal(t, "new-name", seen.body.Name) +} + +// Every deploy walks this path, so an unchanged declaration must stay silent. +// Pushing regardless would write to the service on every `azd up`. +func TestPushMutableIsSilentWhenNothingChanged(t *testing.T) { + held := eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "wording"}, + } + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", project.Eval{ + Name: "support-gate", + Description: "wording", + }, &held) + + assert.Nil(t, seen.body, "an unchanged eval must not be written to") +} + +// Deleting the line from the config is an edit like any other. +func TestPushMutableClearsARemovedDescription(t *testing.T) { + held := eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaDescription: "wording that was deleted"}, + } + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", + project.Eval{Name: "support-gate"}, &held) + + require.NotNil(t, seen.body) + assert.NotContains(t, seen.body.Metadata, metaDescription) +} + +// The update replaces metadata rather than merging it, so anything the service +// or another writer put there has to be carried across or it is dropped. +func TestWithDescriptionKeepsMetadataItDoesNotOwn(t *testing.T) { + held := map[string]string{ + metaEvalName: "support-gate", + metaAgent: "support-agent", + "service_added": "keep me", + } + + merged := withDescription(held, "new wording") + + assert.Equal(t, "new wording", merged[metaDescription]) + assert.Equal(t, "keep me", merged["service_added"]) + assert.Equal(t, "support-agent", merged[metaAgent]) + assert.NotContains(t, held, metaDescription, "the held map must not be mutated") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go new file mode 100644 index 00000000000..4ae09cd3446 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import "regexp" + +// assetNamePattern is what the service accepts for a dataset name. Its own +// refusal is a 400 carrying four levels of nested JSON, and the sentence that +// matters is at the bottom of it. +// +// This extension carries its own copy of the dataset commands, so it needs its +// own copy of the guard: without it the same mistyped name is refused clearly +// by `azd ai dataset` and obscurely by `azd ai eval dataset`. +var assetNamePattern = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +const assetNameMaxLength = 255 + +// validAssetName reports whether the service will accept this name, so a +// mistyped one is refused before a round trip rather than after. +func validAssetName(name string) bool { + if name == "" || len(name) > assetNameMaxLength { + return false + } + return assetNamePattern.MatchString(name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go new file mode 100644 index 00000000000..e4e7b43dc0f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// azd reports "there is no environment" as an ERROR, not as an empty answer: +// +// if defaultEnvironment == "" { +// return nil, environment.ErrDefaultEnvironmentNotFound +// } +// +// So a check that treats every error as "could not ask" can never conclude +// there is no environment, and the diagnostic that names `azd env new` is +// unreachable -- which is what the first version of this did. Equally, treating +// every error as "no environment" would tell someone whose azd hiccupped to +// create an environment they already have. +// +// The only caller reads these off a gRPC call, so every error carries a status. +// These are the shapes that reach it. +func TestNoDefaultEnvironmentIsToldApartFromAFailureToAsk(t *testing.T) { + // The text of azd's environment.ErrDefaultEnvironmentNotFound. + const azdText = "default environment not found" + + assert.True(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, azdText)), + "the sentinel, as azd wraps it in a gRPC status") + assert.True(t, + isNoDefaultEnvironmentError(fmt.Errorf("getting environment: %w", + status.Error(codes.Unknown, azdText))), + "and wrapped again by a caller") + // Outside a project there is nowhere an id could have been recorded, which + // is the same answer for this caller. Missing it told anyone running + // standalone to publish an eval that may already exist. + assert.True(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`")), + "outside a project there is no environment either") + + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.Unavailable, "connection refused")), + "azd being unreachable is not an answer about environments") + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.DeadlineExceeded, "context deadline exceeded")), + "nor is a timeout") + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, "loading project state: permission denied")), + "nor is a daemon that broke while looking") + assert.False(t, isNoDefaultEnvironmentError(nil), "nor is success") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go new file mode 100644 index 00000000000..760844acff4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "text/tabwriter" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +const outputJSON = "json" + +// writePortalLink closes a detail view with the asset's portal URL. +// +// Last line and cyan, matching the sibling extensions, and silent when there is +// no URL — the link is a convenience on top of work already done, so its +// absence must not look like a failure. +func writePortalLink(w io.Writer, url string) { + if url == "" { + return + } + fmt.Fprint(w, messages.PortalLink(color.CyanString(url))) +} + +// runLink is the one link a run has. +// +// The service sends report_url and the extension builds its own portal URL, and +// the two resolve to the same page. Printing both put two labels on one +// destination with no rule a reader could infer, so the service's value wins and +// ours is the fallback that keeps the link from going missing. Callers format +// it themselves, because the three views that show it are laid out differently. +func runLink(reportURL, portalURL string) string { + if reportURL != "" { + return reportURL + } + return portalURL +} + +// outputFormat reads the inherited -o/--output flag. +func outputFormat(cmd *cobra.Command) string { + if cmd == nil { + return "" + } + v, err := cmd.Flags().GetString("output") + if err != nil { + return "" + } + return strings.ToLower(v) +} + +// isJSON reports whether the command should emit machine-readable output. +func isJSON(cmd *cobra.Command) bool { + return outputFormat(cmd) == outputJSON +} + +// noPrompt reports whether the caller asked for no interaction. +// +// JSON output counts: a prompt written into a document nobody is reading is a +// hang, not a question. +func noPrompt(cmd *cobra.Command) bool { + if isJSON(cmd) { + return true + } + value, err := cmd.Flags().GetBool("no-prompt") + return err == nil && value +} + +// commandContext is cmd.Context() with cobra's pre-Execute nil made safe. +// gRPC dereferences the context it is handed, so a nil one is a panic rather +// than a failed call. +func commandContext(cmd *cobra.Command) context.Context { + if ctx := cmd.Context(); ctx != nil { + return ctx + } + return context.Background() +} + +// emitJSON writes v as indented JSON. +func emitJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// emitJSONList writes items as a JSON array. +// +// List commands emit a bare array rather than the envelope the service replied +// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap +// results in `data`, the ARM-shaped ones in `value` — so passing them through +// would make a caller's parsing depend on which service happens to back a given +// command. They also carry paging fields that this extension does not follow, +// which would suggest there is more to fetch when there is not. +// +// A nil slice encodes as `null`, so it is normalized to an empty array: a +// caller iterating the result should see no elements, not a type error. +func emitJSONList[T any](w io.Writer, items []T) error { + if items == nil { + items = []T{} + } + return emitJSON(w, items) +} + +// emitTable writes a list view: uppercase headers over a rule, tab-aligned. +// +// The rule is what separates the header from the data at a glance, and it is +// what `azure.ai.skills` prints, so a reader moving between the Foundry +// extensions sees one table. +func emitTable(w io.Writer, headers []string, rows [][]string) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { + return err + } + rule := make([]string, len(headers)) + for i, h := range headers { + rule[i] = strings.Repeat("-", len(h)) + } + if _, err := fmt.Fprintln(tw, strings.Join(rule, "\t")); err != nil { + return err + } + for _, row := range rows { + if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { + return err + } + } + return tw.Flush() +} + +// field is one row of a detail view. +type field struct { + Key string // Title Case, per the azd style guide + Value string +} + +// emitDetail writes a two-column key/value view, the shape `show` uses. +// +// Empty values are dropped rather than printed blank: a detail view is read to +// learn what a thing is, and a column of empty keys says only that the writer +// did not know which fields this kind has. +func emitDetail(w io.Writer, fields []field) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + for _, f := range fields { + if f.Value == "" { + continue + } + if _, err := fmt.Fprintf(tw, "%s\t%s\n", f.Key, f.Value); err != nil { + return err + } + } + return tw.Flush() +} + +// requireFlag returns an error naming a flag the command needs and has no way +// to settle for itself. +func requireFlag(name string) error { + return messages.FlagRequired(name) +} + +// writeFileAtomic replaces a file's contents in one step. +// +// The caller is usually overwriting a definition the developer already has and +// wants to keep working with, so a half-written file is worse than no write at +// all: os.WriteFile truncates first, and a failure after that leaves the good +// local copy destroyed. +// +// Every error names the path the caller passed. The temporary file is this +// function's business and appears nowhere the caller asked for. +func writeFileAtomic(path string, body []byte) error { + // Refuse anything that is not a regular file: pointed at a directory, the + // replacement below would report a confusing rename failure instead. + switch info, err := os.Stat(path); { + case err == nil && !info.Mode().IsRegular(): + return messages.NotARegularFile(path) + case err != nil && !errors.Is(err, os.ErrNotExist): + return messages.Creating(path, err) + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".azd-eval-*") + if err != nil { + return messages.CannotWriteInDirectory(dir, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() + return messages.Creating(path, err) + } + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return messages.Creating(path, err) + } + if err := tmp.Close(); err != nil { + return messages.Creating(path, err) + } + if err := project.ReplaceFile(tmpName, path); err != nil { + return messages.Creating(path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go new file mode 100644 index 00000000000..38b9e3824ac --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The list commands are backed by two different services whose envelopes +// disagree — `data` on one side, `value` on the other. Emitting whichever one +// came back would make a caller's parsing depend on that accident, so every +// list emits a bare array instead. +func TestEmitJSONList_EmitsAnArrayNotAnEnvelope(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSONList(&buf, []string{"a", "b"})) + assert.Equal(t, "[\n \"a\",\n \"b\"\n]\n", buf.String()) +} + +// A nil slice marshals to `null`, which a caller iterating the output cannot +// range over. An empty listing has to come back as an empty array. +func TestEmitJSONList_NilBecomesEmptyArray(t *testing.T) { + var buf bytes.Buffer + var none []string + require.NoError(t, emitJSONList(&buf, none)) + assert.Equal(t, "[]\n", buf.String()) +} + +// `evaluator show --output-file` is pointed at a definition the developer is +// still working with, so a write that cannot complete must leave the old one +// intact rather than truncate it. +func TestWriteFileAtomic(t *testing.T) { + t.Run("replaces an existing file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "evaluator.json") + require.NoError(t, os.WriteFile(path, []byte("old"), 0o600)) + require.NoError(t, writeFileAtomic(path, []byte("new"))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new", string(body)) + }) + + t.Run("creates a file that was not there", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "evaluator.json") + require.NoError(t, writeFileAtomic(path, []byte("new"))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new", string(body)) + }) + + t.Run("a directory is refused, not removed", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "evaluator.json") + require.NoError(t, os.Mkdir(path, 0o750)) + + require.Error(t, writeFileAtomic(path, []byte("new"))) + + info, err := os.Stat(path) + require.NoError(t, err, "the directory must survive") + assert.True(t, info.IsDir()) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Len(t, entries, 1, "no temporary file may be left behind") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go new file mode 100644 index 00000000000..f4173a516e9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The portal link is the last line of a detail view, and it is the one thing a +// user clicks to see the run they just waited for. +func TestWritePortalLink(t *testing.T) { + var buf bytes.Buffer + writePortalLink(&buf, "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r") + + out := buf.String() + assert.Contains(t, out, "Portal: ") + assert.Contains(t, out, "/build/evaluations/e/run/r") + assert.True(t, strings.HasSuffix(out, "\n"), "it closes the view, so it ends the line") +} + +// Resolution is best effort: the link is a convenience on top of work already +// done, so having none must print nothing rather than an empty label that +// reads like a failure. +func TestWritePortalLink_SilentWithoutAURL(t *testing.T) { + var buf bytes.Buffer + writePortalLink(&buf, "") + + assert.Empty(t, buf.String()) +} + +// Colour is pinned off for the rest of the package, which leaves nothing +// exercising the branch that actually runs in a terminal. The escape codes have +// to wrap the URL and nothing else: one leaking into the label, or past the +// newline, follows the link into whatever a reader pastes it in. +func TestWritePortalLink_WrapsOnlyTheURL(t *testing.T) { + restore := color.NoColor + color.NoColor = false + t.Cleanup(func() { color.NoColor = restore }) + + var buf bytes.Buffer + writePortalLink(&buf, "https://ai.azure.com/x") + + assert.Equal(t, "Portal: \x1b[36mhttps://ai.azure.com/x\x1b[0m\n", buf.String()) +} + +// `-o json` carries the same link the terminal prints, so a pipeline reading +// JSON is not the one consumer that cannot find the run in the portal. +func TestRunPortalURLTravelsInJSON(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/eval_1/run/evalrun_1", + } + + var buf bytes.Buffer + require.NoError(t, emitJSON(&buf, run)) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + assert.Equal(t, run.PortalURL, decoded["portal_url"], + "the key is portal_url, which is what the spec tells consumers to read") +} + +// A run with no portal link must not carry an empty key, or a consumer cannot +// tell "no link" from "link is the empty string". +func TestRunWithoutPortalURLOmitsTheKey(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSON(&buf, &eval_api.OpenAIEvalRun{ID: "evalrun_1"})) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + assert.NotContains(t, decoded, "portal_url") +} + +// The portal URL is built from the eval and run ids, which is what makes the +// link land on the run rather than the eval's list of them. +func TestPortalRunURLShape(t *testing.T) { + prefix, err := eval_api.NewPortalPrefix( + "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + assert.True(t, strings.HasSuffix( + prefix.EvalRunURL("eval_1", "evalrun_9"), + "/build/evaluations/eval_1/run/evalrun_9")) +} + +// A run has one destination. The service's report_url and the portal URL the +// extension builds resolve to the same page, and printing both put two labels +// on it with no rule a reader could infer. +func TestRenderRunPrintsOneLink(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + ReportURL: "https://service.example/report/1", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r", + } + + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, run, nil)) + + out := buf.String() + assert.Contains(t, out, "Report: https://service.example/report/1", + "the service's url wins where it sent one") + assert.NotContains(t, out, "Portal: ", + "the second label named the same destination") + assert.NotContains(t, out, run.PortalURL) +} + +// Ours is the fallback, so a service that sends no report_url does not leave +// the reader with no way to open the run. +func TestRenderRunFallsBackToTheBuiltLink(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r", + } + + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, run, nil)) + + assert.Contains(t, buf.String(), "Report: "+run.PortalURL) +} + +// A run with neither prints no link rather than an empty label. +func TestRenderRunOmitsAnAbsentLink(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", Status: "completed", + }, nil)) + + assert.NotContains(t, buf.String(), "Report:") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go new file mode 100644 index 00000000000..411a9c3f377 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func relevanceSchema(required ...string) map[string]*eval_api.EvaluatorSummary { + return map[string]*eval_api.EvaluatorSummary{ + "builtin.relevance": { + Name: "builtin.relevance", + SupportedEvaluationLevels: []string{"turn"}, + Definition: &eval_api.EvaluatorContract{ + InitParameters: &eval_api.JSONSchema{Required: required}, + }, + }, + } +} + +func evalRequiring(params map[string]any) *project.Eval { + return &project.Eval{ + Name: "quality", + Dataset: "golden", + EvaluationLevel: "turn", + Evaluators: evalcore.EvaluatorList{{ + Evaluator: "builtin.relevance", + InitializationParameters: params, + }}, + } +} + +// The same requirement was checked while building the request, which runs after +// the datasets and evaluators have been pushed. A missing judge deployment +// therefore cost an immutable dataset version per attempt, and the version +// number climbs whether or not the eval is ever created. +func TestCreateRefusesAMissingJudgeBeforePublishing(t *testing.T) { + err := checkEvaluatorRequirements(evalRequiring(nil), relevanceSchema("deployment_name")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "deployment_name") + assert.Contains(t, err.Error(), "builtin.relevance") +} + +// Either spelling of the judge deployment satisfies it, the same way the +// request builder binds whichever the evaluator publishes. +func TestEitherJudgeSpellingSatisfiesTheRequirement(t *testing.T) { + for _, declared := range []string{"deployment_name", "model"} { + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{declared: "o4-mini"}), + relevanceSchema("deployment_name")) + + assert.NoErrorf(t, err, "%q is the same parameter under the other name", declared) + } +} + +// evaluation_level is supplied from the eval's own declaration rather than +// written under initialization_parameters, so requiring it must not refuse a +// configuration that sets the level. +func TestRequiredEvaluationLevelComesFromTheDeclaration(t *testing.T) { + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{"deployment_name": "o4-mini"}), + relevanceSchema("deployment_name", "evaluation_level")) + + assert.NoError(t, err) +} + +// An evaluator the listing did not describe leaves the service with the last +// word, which is what happened before this check existed. +func TestAnUnknownEvaluatorIsLeftToTheService(t *testing.T) { + err := checkEvaluatorRequirements(evalRequiring(nil), + map[string]*eval_api.EvaluatorSummary{}) + + assert.NoError(t, err, "nothing published to check against") +} + +// A level the evaluator does not support is the other thing settled by the +// contract alone, so it is worth catching before a publish too. +func TestAnUnsupportedLevelIsRefusedBeforePublishing(t *testing.T) { + schemas := relevanceSchema("deployment_name") + schemas["builtin.relevance"].SupportedEvaluationLevels = []string{"conversation"} + + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{"deployment_name": "o4-mini"}), schemas) + + require.Error(t, err) + assert.Contains(t, err.Error(), "turn") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go new file mode 100644 index 00000000000..2588ad49b9c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +// TestConfigureExtensionHostMatchesManifest verifies that the providers this +// extension registers match those declared in its extension.yaml. +func TestConfigureExtensionHostMatchesManifest(t *testing.T) { + manifestPath := filepath.Join("..", "..", "extension.yaml") + require.NoError(t, azdext.VerifyProvidersMatchManifest(configureExtensionHost, manifestPath)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go new file mode 100644 index 00000000000..521f5210022 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -0,0 +1,730 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "io/fs" + "log" + "maps" + "os" + "reflect" + "strconv" + "strings" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" +) + +// evalReconciler applies the eval configuration to the data plane. It is the +// deploy half of the provider; the provider owns ordering, this owns the calls. +type evalReconciler struct { + ec *evalContext +} + +var _ project.Reconciler = (*evalReconciler)(nil) + +func newEvalReconciler(ctx context.Context) (project.Reconciler, error) { + ec, err := newEvalContext(ctx, "") + if err != nil { + return nil, err + } + return &evalReconciler{ec: ec}, nil +} + +// EnsureDataset registers a new version only when the local content changed. +// +// The dataset API exposes no content hash, so comparing against the service +// would mean downloading the blob on every deploy. Instead the local file is +// hashed and the digest kept in the azd environment. +func (r *evalReconciler) EnsureDataset( + ctx context.Context, + decl project.DatasetDecl, + localPath string, +) (string, bool, error) { + // No local source means the dataset is already registered; just confirm it. + if localPath == "" { + version := decl.Version + if version == "" { + list, err := r.ec.datasetClient.ListDatasetVersions( + ctx, decl.Name, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, messages.DatasetNotLocalNorFound(decl.Name, err) + } + if len(list.Value) == 0 { + return "", false, messages.DatasetNotLocalNorRegistered(decl.Name) + } + version = dataset_api.LatestVersion(list.Value) + } else if _, err := r.ec.datasetClient.GetDataset( + ctx, decl.Name, version, ProjectEndpointAPIVersion, + ); err != nil { + return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, version) + } + + // Recorded so a run reads the version reconciliation settled on. Without + // this a pin is honoured at deploy and then ignored at run time, which + // scores different rows than the ones the author asked for. + r.ec.remember(ctx, versionKey("dataset", decl.Name), version) + return version, false, nil + } + + if _, err := os.Stat(localPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, messages.DatasetNotGeneratedYet(decl.Name, localPath) + } + return "", false, messages.DatasetSource(localPath, err) + } + + // A malformed row is only noticed once the service tries to evaluate it, + // by which point a version has been published and the eval points at + // it. Reading the file here costs nothing and names the offending line. + if err := validateJSONL(localPath); err != nil { + return "", false, messages.DatasetProblem(decl.Name, err) + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("dataset", decl.Name) + if prior := r.ec.getEnvValue(ctx, key); prior == digest { + // Unchanged since the last deploy; reuse the recorded version, but only + // after confirming nobody published a newer one outside the repo. An + // explicit `version:` is the author saying which version they want, so + // it settles the question and the check does not apply. + if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + if decl.Version != "" { + // A pin settles which version to use, not whether it is still + // there. Skipping the service entirely let a deleted version + // report as unchanged while the eval pointed at nothing. Only a + // confirmed 404 refuses: anything else leaves the pin alone + // rather than failing a deploy on a transient read. + if _, err := r.ec.datasetClient.GetDataset( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ); err != nil && dataset_api.IsNotFound(err) { + return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, decl.Version) + } + return decl.Version, false, nil + } + if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { + return "", false, err + } + return version, false, nil + } + } + + // Uploaded by the path the author declared. Collapsing a file to its + // directory would upload whichever .jsonl sorts first, while the + // fingerprint below still describes the declared one. + dir := localPath + + // A declared version is the version to publish, not one to count from. + // Reaching here means the content differs from what that version holds, so + // republishing over it would change a version the author pinned. + if decl.Version != "" { + ds, err := r.ec.datasetClient.UploadVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + if dataset_api.IsVersionConflict(err) { + return "", false, messages.DatasetVersionConflict(decl.Name, decl.Version) + } + return "", false, err + } + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, versionKey("dataset", decl.Name), ds.Version) + return ds.Version, true, nil + } + + // UploadNextVersion discovers the currently registered version when none is + // declared, so the upload does not restart at 1.0 and collide. + ds, err := r.ec.datasetClient.UploadNextVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, versionKey("dataset", decl.Name), ds.Version) + r.ec.remember(ctx, envKeyDatasetVersion, ds.Version) + + return ds.Version, true, nil +} + +// validateJSONL checks that every row is a JSON object before the file is +// published. +// +// The service accepts the upload whatever the bytes are, so a typo becomes a +// registered version, an eval bound to it, and a run that fails on a row +// nobody has looked at. Blank lines are skipped: they are not rows. +func validateJSONL(path string) error { + f, err := os.Open(path) + if err != nil { + return messages.ReadingPath(path, err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + rows := 0 + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return messages.JSONLRowInvalid(path, line, err) + } + if len(row) == 0 { + return messages.JSONLRowEmpty(path, line) + } + rows++ + } + if err := scanner.Err(); err != nil { + return messages.ReadingPath(path, err) + } + if rows == 0 { + return messages.JSONLNoRows(path) + } + return nil +} + +func (r *evalReconciler) checkDatasetDrift( + ctx context.Context, + name, recorded string, +) error { + latest, err := r.latestDatasetVersion(ctx, name) + if err != nil { + // The whole point of this check is to catch a version published behind + // our back. A listing we could not read is not evidence there was none. + return err + } + if latest == "" || latest == recorded { + return nil + } + if !dataset_api.VersionGreater(latest, recorded) { + return nil + } + return messages.DatasetDrifted(name, latest, recorded) +} + +// latestDatasetVersion reports the newest registered version. A dataset the +// service does not know, and a listing that has not caught up, both report an +// empty version and no error; anything else is returned. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) (string, error) { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + if dataset_api.IsNotFound(err) { + return "", nil + } + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", nil + } + return dataset_api.LatestVersion(list.Value), nil +} + +// EnsureEvaluator publishes a new version when the local definition differs +// from what the service holds. +// +// The two kinds of evaluator are told apart by the source's extension: `.py` +// is code, anything else is a rubric. They also detect change differently. A +// rubric definition comes back inline, so it is compared directly; a code +// definition's source is not read back in a form worth comparing, so a +// fingerprint of the script is kept in the azd environment, the same way +// datasets work. +func (r *evalReconciler) EnsureEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + localPath string, +) (string, bool, error) { + if localPath == "" { + raw, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, messages.EvaluatorNotLocalNorFound(decl.Name, err) + } + return versionFromRaw(raw, decl.Version), false, nil + } + + if _, err := os.Stat(localPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, messages.EvaluatorNotGeneratedYet(decl.Name, localPath) + } + return "", false, messages.EvaluatorSource(localPath, err) + } + + raw, err := project.ReadFileNoBOM(localPath) + if err != nil { + return "", false, messages.EvaluatorSource(localPath, err) + } + + body, err := normalizeRubricBody(decl.Name, raw) + if err != nil { + return "", false, messages.EvaluatorProblem(decl.Name, err) + } + + // The author's own file decides whether there is anything to publish. + // Comparing against the service cannot: it enriches a definition with + // fields nobody authored, so sameDefinition only looks for authored keys on + // the service and a key the author *deleted* — a pass_threshold, say — is + // still there to be found, and the deletion never publishes. + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, messages.EvaluatorSource(localPath, err) + } + digestKey := project.FingerprintKey("evaluator", decl.Name) + prior := r.ec.getEnvValue(ctx, digestKey) + authorEdited := prior != "" && prior != digest + + // Compare against the definition already on the service. + var known json.RawMessage + existing, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, "", ProjectEndpointAPIVersion, + ) + // A read that failed is not a read that found nothing: falling through + // publishes a new version with no drift check, over whatever is already + // there. Only a confirmed absence is a first publish. + if err != nil && !eval_api.IsNotFound(err) { + return "", false, messages.CheckingEvaluatorExists(decl.Name, err) + } + if err == nil { + remote := versionFromRaw(existing, "") + if !authorEdited && sameDefinition(existing, body) { + // Nothing to publish, but the version is still worth recording: + // it is what a later deploy compares against to notice that + // someone moved the evaluator on from here. + if remote != "" { + r.ec.remember(ctx, versionKey("evaluator", decl.Name), remote) + } + r.ec.remember(ctx, digestKey, digest) + return versionFromRaw(existing, decl.Version), false, nil + } + + // The definitions differ, which means either the local file changed + // or someone published a version outside the repo. The version + // recorded at the last deploy is what tells them apart, and + // publishing over the second case would bury an intentional change + // under one nobody asked for. + if recorded := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)); recorded != "" { + if err := checkEvaluatorDrift(decl.Name, recorded, remote); err != nil { + return "", false, err + } + } + + // What that read saw is what keeps the publish from being answered + // with it again. + known = existing + } + + created, err := r.ec.evalClient.CreateEvaluatorVersion( + ctx, decl.Name, body, known, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + r.ec.remember(ctx, versionKey("evaluator", decl.Name), created.Version) + r.ec.remember(ctx, digestKey, digest) + return created.Version, true, nil +} + +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// It is asked only when the local definition and the remote one disagree, +// which on its own says nothing about who moved: the author may have edited +// the file, or someone may have published a version from outside the repo. +// The recorded version settles it, and the difference matters because +// publishing is how this reconciler resolves a disagreement — doing that over +// a version somebody deliberately published would bury their change under one +// nobody asked for, with `azd up` reporting success. +// +// The remote version is passed in rather than listed, because the version +// listing lags a publish and would report an evaluator as un-drifted for the +// first seconds of its newest version's life. +func checkEvaluatorDrift(name, recorded, remote string) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + remoteNumber, err := strconv.Atoi(remote) + if err != nil || remoteNumber <= recordedNumber { + return nil + } + return messages.EvaluatorDrifted(name, remote, recorded) +} + +// evaluatorPropagation bounds the wait for a freshly published evaluator to +// become usable. +// +// A create returns before the version is resolvable everywhere, and the very +// next step of a deploy is EnsureEval, which names the evaluator in a testing +// criterion. Creating the eval inside that window fails with "The evaluator X +// was not found" — a confusing error, because the evaluator was published +// seconds earlier and is plainly there by the time anyone looks. The observed +// gap is under a second, so the poll is frequent and the cap is generous +// enough to absorb a slow day without stalling a deploy on an evaluator that +// is genuinely missing. +const ( + evaluatorPropagationTimeout = 30 * time.Second + evaluatorPropagationInterval = 250 * time.Millisecond +) + +// awaitEvaluatorReadable polls until a published version is resolvable, or the +// cap passes. +// +// Two reads have to agree, because they are not backed by the same view. The +// direct read goes consistent almost immediately; the version listing lags it +// by seconds, the same way the dataset listing does. A live publish was +// observed reading back at 03:06:58 and still failing eval creation at +// 03:06:59, so waiting on the direct read alone leaves exactly the race this +// exists to close. The listing is the slower of the two and therefore the one +// worth waiting on. +// +// A timeout is not an error. The wait is a courtesy that makes the common case +// reliable; if it never succeeds, the create that follows will report the real +// problem with far more context than a wait that gave up could. +func (r *evalReconciler) awaitEvaluatorReadable(ctx context.Context, name, version string) { + if version == "" { + return + } + deadline := time.Now().Add(evaluatorPropagationTimeout) + for { + if r.evaluatorVersionResolvable(ctx, name, version) { + return + } + if time.Now().After(deadline) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(evaluatorPropagationInterval): + } + } +} + +// evaluatorVersionResolvable reports whether a version can be both read +// directly and found in the listing. +func (r *evalReconciler) evaluatorVersionResolvable( + ctx context.Context, + name, version string, +) bool { + if _, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return false + } + + list, err := r.ec.evalClient.ListEvaluatorVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + +// EnsureEval creates the eval when it has never been deployed, or when its own +// declaration changed. Evals are immutable, so a declaration change means a new +// eval and a new id. +// +// What an eval's references *resolve to* is deliberately not a reason to +// recreate it: an evaluator tracking latest that publishes a new version leaves +// every eval that runs it alone, which is what keeps a rubric edit comparable +// against the runs taken before it. +func (r *evalReconciler) EnsureEval( + ctx context.Context, + group project.Eval, + datasetPath string, +) (string, bool, error) { + if group.ID != "" { + return group.ID, false, nil + } + + // Evals are immutable, so a change to the eval's own substance — evaluators, + // dataset, source, target, level — needs a new eval. Name and description are + // excluded from the digest and pushed in place instead. + recreate := false + digest, err := project.FingerprintGroup(group) + if err != nil { + return "", false, err + } + key := project.FingerprintKey("eval", group.Name) + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { + recreate = true + } + + // Building the request is also what checks the declaration against the + // dataset's columns, so it happens before the reuse decision: a dataset can + // lose a column an evaluator needs without the eval's own declaration + // changing, and reusing the eval would let that reach a run unreported. + req, err := buildEvalRequest( + &group, + r.ec.evaluatorSchemas(ctx), + datasetColumnsFromPath(datasetPath), + ) + if err != nil { + return "", false, err + } + + cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) + if cached == "" && !recreate { + // Nothing recorded under this name, but the substance may already be + // deployed under the name it had before. The environment records the id + // against the digest as well, which is what recognizes a rename rather + // than reading it as a delete plus an add. + adopted, err := r.adoptRenamed(ctx, group, digest) + if err != nil { + return "", false, err + } + if adopted != "" { + cached = adopted + } + } + if cached != "" && !recreate { + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, cached) + if err != nil && !eval_api.IsNotFound(err) { + // A read that failed is not an eval that is gone. Falling through + // on a 429, a 503 or an expired token would create a second eval + // and overwrite the recorded id, forking for good the run history + // this lookup exists to keep. + return "", false, err + } + if err == nil { + // Reusing the eval is not the same as leaving it alone: name and + // description are excluded from the digest because they must not + // split a history, which makes this the only place an edit to + // either of them can reach the service. + r.pushMutable(ctx, cached, group, remote) + + // Record the digest on reuse as well, otherwise an eval deployed + // before fingerprinting existed never establishes a baseline and + // later edits go undetected. + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, idKey("eval", group.Name), cached) + r.ec.remember(ctx, digestIDKey(digest), cached) + r.ec.remember(ctx, envKeyEvalID, cached) + return cached, false, nil + } + } + + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) + if err != nil { + return "", false, err + } + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, idKey("eval", group.Name), created.ID) + r.ec.remember(ctx, digestIDKey(digest), created.ID) + // EVAL_ID stays the last-deployed eval. Nothing reads it to decide which + // eval a command means, because every deploy writes it and it cannot say + // which declaration it belongs to; it is here for anything outside this + // extension that wants the id of what was just deployed. + r.ec.remember(ctx, envKeyEvalID, created.ID) + return created.ID, true, nil +} + +// adoptRenamed reclaims the eval this declaration used to be called, so a +// rename keeps the id and every run under it rather than forking the history. +// +// The name is what UpdateEvalParametersBody reaches, so the new one is pushed +// to the service. +func (r *evalReconciler) adoptRenamed( + ctx context.Context, + group project.Eval, + digest string, +) (string, error) { + id := r.ec.getEnvValue(ctx, digestIDKey(digest)) + if id == "" { + return "", nil + } + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) + if err != nil { + if eval_api.IsNotFound(err) { + // The eval it used to be called is genuinely gone, so there is + // nothing to adopt and the caller creates one. + return "", nil + } + return "", err + } + r.pushMutable(ctx, id, group, remote) + return id, nil +} + +// pushMutable sends the half of a declaration the service treats as mutable. +// +// Substance never travels this way — an edit that touches it is a new eval. +// Name and description are left out of the fingerprint precisely because they +// cost nothing to change and must not split a run history, so they are +// reconciled here rather than ignored, and the eval keeps its id and every run +// under it. +// +// A failure is not fatal. The eval is still the right one and the declaration +// still resolves; it just reads under its old wording in the portal until the +// next deploy. +func (r *evalReconciler) pushMutable( + ctx context.Context, + id string, + group project.Eval, + remote *eval_api.OpenAIEval, +) { + if remote == nil { + return + } + desired := withDescription(remote.Metadata, group.Description) + if remote.Name == group.Name && maps.Equal(remote.Metadata, desired) { + return + } + if _, err := r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ + Name: group.Name, + Metadata: desired, + }); err != nil { + // Deliberately not fatal: a name or description that did not travel + // leaves the eval usable, and failing the deploy over it would be + // worse. It still has to be findable, so --debug can see it. + log.Printf("[reconcile] updating eval %s name/description: %v", id, err) + } +} + +// withDescription applies the declaration's description to the metadata the +// service already holds, leaving every other key alone — including any the +// service added itself, which a replacing update would otherwise drop. +func withDescription(held map[string]string, description string) map[string]string { + merged := make(map[string]string, len(held)+1) + maps.Copy(merged, held) + if description == "" { + delete(merged, metaDescription) + } else { + merged[metaDescription] = description + } + return merged +} + +// sameDefinition reports whether the locally authored definition already +// matches what the service holds. +// +// Only the keys the candidate declares are compared. The service enriches a +// definition when it is created — a rubric of nothing but `type` and +// `dimensions` comes back carrying data_schema, init_parameters and metrics it +// was never given — so comparing whole documents never matches and every +// deploy publishes a redundant version. +func sameDefinition(existing, candidate []byte) bool { + extract := func(raw []byte) map[string]json.RawMessage { + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return nil + } + def, ok := doc["definition"] + if !ok { + return nil + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(def, &fields); err != nil { + return nil + } + return fields + } + + onService, authored := extract(existing), extract(candidate) + if onService == nil || authored == nil { + return false + } + + for key, want := range authored { + got, ok := onService[key] + if !ok || !equalJSON(got, want) { + return false + } + } + return true +} + +// equalJSON compares two JSON values structurally, so key order and +// whitespace do not register as a change. +func equalJSON(a, b json.RawMessage) bool { + var left, right any + if err := json.Unmarshal(a, &left); err != nil { + return false + } + if err := json.Unmarshal(b, &right); err != nil { + return false + } + return reflect.DeepEqual(left, right) +} + +func versionFromRaw(raw []byte, fallback string) string { + var doc struct { + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &doc); err == nil && doc.Version != "" { + return doc.Version + } + return fallback +} + +// versionKey holds the version resolved for an artifact at the last deploy. +func versionKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_VERSION" +} + +// recordDeployedDataset records the state a deploy would have left behind for a +// dataset that the service has already registered and that already has a local +// copy. +// +// `azd up` decides whether to publish by comparing the local file against a +// fingerprint held in the environment. A generated dataset arrives with no such +// fingerprint, so without this the first deploy after `generate` reads the file +// as new and publishes a second version identical to the one the job just +// registered. Only a local edit should produce version 2. +// +// Best effort: failing to record costs a redundant version, not correctness. +func (ec *evalContext) recordDeployedDataset( + ctx context.Context, + name, localPath, version string, +) { + digest, err := project.Fingerprint(localPath) + if err != nil { + return + } + ec.remember(ctx, project.FingerprintKey("dataset", name), digest) + if version != "" { + ec.remember(ctx, versionKey("dataset", name), version) + } +} + +// idKey names the env entry holding a resolved id. +// +// Ids are per declaration. A single shared key works only while a config has +// one group: with two, the second deploy finds the first's id cached, confirms +// it exists, and hands it back for the wrong group. +func idKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_ID" +} + +// digestIDKey records an eval's id against its substance, which is what lets a +// renamed declaration find the eval it already deployed. Keyed by a prefix of +// the digest, because the whole hash makes an unreadable environment variable. +func digestIDKey(digest string) string { + return "EVAL_SUBSTANCE_" + strings.ToUpper(digest[:16]) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go new file mode 100644 index 00000000000..488feccf38f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Drift is only interesting when the two definitions already disagree, and +// then only when the disagreement came from the project rather than from the +// author. These are the four ways that question can be answered. +func TestCheckEvaluatorDrift(t *testing.T) { + // The author edited the file. The project is where the last deploy left + // it, so publishing is exactly right and must not be blocked. + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "3")) + + // Someone published outside the repo. Publishing over it would leave + // their change behind with `azd up` reporting success. + err := checkEvaluatorDrift("support-quality", "3", "4") + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + assert.Contains(t, err.Error(), "version 4") + assert.Contains(t, err.Error(), "3 was recorded") + assert.Contains(t, err.Error(), "outside this configuration", + "the message has to say what moved, not just that something did") + + // A version that went backwards is not drift: a newer version was + // deleted, and republishing is how the repo takes the name back. + require.NoError(t, checkEvaluatorDrift("support-quality", "4", "3")) + + // Versions this extension did not number cannot be compared, and refusing + // a deploy over a numbering convention it does not own would be worse + // than not checking. + require.NoError(t, checkEvaluatorDrift("support-quality", "", "4")) + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "preview")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go new file mode 100644 index 00000000000..a29598c0db4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// testEnvServer is an azd environment held in memory, so reconciliation paths +// that only run once something was recorded at the last deploy are reachable +// from a test. Without it, getEnvValue answers "" for everything and those +// branches never execute. +type testEnvServer struct { + azdext.UnimplementedEnvironmentServiceServer + values map[string]string +} + +func (s *testEnvServer) GetValue( + _ context.Context, req *azdext.GetEnvRequest, +) (*azdext.KeyValueResponse, error) { + return &azdext.KeyValueResponse{Value: s.values[req.Key]}, nil +} + +func (s *testEnvServer) SetValue( + _ context.Context, req *azdext.SetEnvRequest, +) (*azdext.EmptyResponse, error) { + if s.values == nil { + s.values = map[string]string{} + } + s.values[req.Key] = req.Value + return &azdext.EmptyResponse{}, nil +} + +// newTestAzdClient serves the environment over gRPC the way azd itself does, +// rather than faking the accessor, so the client code under test is the real one. +func newTestAzdClient(t *testing.T, env *testEnvServer) *azdext.AzdClient { + t.Helper() + + server := grpc.NewServer() + azdext.RegisterEnvironmentServiceServer(server, env) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + + client, err := azdext.NewAzdClient(azdext.WithAddress(listener.Addr().String())) + require.NoError(t, err) + t.Cleanup(func() { client.Close() }) + + return client +} + +// pinnedDatasetReconciler builds a reconciler whose environment already holds +// what a previous deploy recorded for a dataset, and whose service answers a +// version read with the given status. +func pinnedDatasetReconciler( + t *testing.T, name, version string, versionStatus int, +) (*evalReconciler, string) { + t.Helper() + + dir := t.TempDir() + localPath := filepath.Join(dir, name+".jsonl") + require.NoError(t, os.WriteFile(localPath, []byte("{\"query\":\"hi\"}\n"), 0o600)) + + digest, err := project.Fingerprint(localPath) + require.NoError(t, err) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/versions/") { + w.WriteHeader(http.StatusNotFound) + return + } + if versionStatus != http.StatusOK { + w.WriteHeader(versionStatus) + return + } + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": name, "version": version, + })) + })) + t.Cleanup(srv.Close) + + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("dataset", name): digest, + versionKey("dataset", name): version, + }} + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + + return &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }}, localPath +} + +// A pin settles which version to use, not whether it is still there. Reusing it +// unread let a deleted version report as unchanged while the eval pointed at +// nothing, which is what `create` did straight after `dataset delete`. +func TestEnsureDatasetRefusesAPinnedVersionTheServiceNoLongerHas(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusNotFound) + + _, _, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.Error(t, err) + assert.Contains(t, err.Error(), "1.0", "the version that is missing") + assert.Contains(t, err.Error(), "versions list", "and the command that shows what is there") +} + +// The ordinary case: the pin is still registered, so reconciliation reuses it +// and reports no change. +func TestEnsureDatasetReusesAPinnedVersionThatStillExists(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusOK) + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1.0", version) + assert.False(t, changed, "an unchanged file at a pinned version publishes nothing") +} + +// A read that failed is not a read that came back empty. Failing the deploy on +// a 403 or a timeout would turn a transient service problem into a broken +// pipeline for a pin that is very probably fine. +func TestEnsureDatasetKeepsAPinnedVersionWhenTheReadFails(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusForbidden) + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1.0", version) + assert.False(t, changed) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go new file mode 100644 index 00000000000..54ced469be9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// The service enriches a definition when it stores it: a rubric of nothing but +// type and dimensions comes back carrying data_schema, init_parameters and +// metrics. Comparing whole documents therefore never matched, and every deploy +// published a redundant version. +func TestSameDefinitionIgnoresServerAddedFields(t *testing.T) { + authored := []byte(`{ + "name": "r", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}] + } + }`) + + onService := []byte(`{ + "name": "r", + "version": "2", + "created_at": "2026-07-28T00:00:00Z", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}], + "data_schema": {"type":"object","properties":{"query":{"type":"string"}}}, + "init_parameters": {"required":["model"],"properties":{"model":{"type":"string"}}}, + "metrics": {"score":{"type":"number"}} + } + }`) + + require.True(t, sameDefinition(onService, authored), + "server-added fields must not count as a change") +} + +// A real edit still registers. +func TestSameDefinitionDetectsAuthoredChange(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":7}]}}`) + onService := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}],"metrics":{}}}`) + + require.False(t, sameDefinition(onService, authored)) +} + +// Key order and whitespace are not changes. +func TestSameDefinitionIsStructural(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte("{\"definition\":{\n \"dimensions\": [ {\"weight\":5,\"id\":\"a\"} ],\n \"type\":\"rubric\"\n}}") + + require.True(t, sameDefinition(onService, authored)) +} + +func TestSameDefinitionRejectsMalformed(t *testing.T) { + good := []byte(`{"definition":{"type":"rubric"}}`) + require.False(t, sameDefinition([]byte(`not json`), good)) + require.False(t, sameDefinition(good, []byte(`not json`))) + require.False(t, sameDefinition([]byte(`{"no":"definition"}`), good)) +} + +// What sameDefinition cannot see, and why EnsureEvaluator digests the author's +// file instead of relying on it. +// +// The comparison walks the authored keys and looks for each on the service. A +// key the author *deleted* is not among them, so its survival on the service +// goes unnoticed and the definitions are called equal. Deleting a +// pass_threshold — the spec's own Scenario 4 edit, in reverse — would publish +// nothing and leave the old threshold grading every run. +func TestSameDefinitionCannotSeeARemovedField(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte( + `{"definition":{"type":"rubric","pass_threshold":0.7,` + + `"dimensions":[{"id":"a","weight":5}]}}`) + + require.True(t, sameDefinition(onService, authored), + "this is the blind spot the digest exists to cover, not a property to rely on") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go new file mode 100644 index 00000000000..fa417ce43c3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// Precedence decides behaviour without announcing it, so a wrong answer here +// is silent. options.max_samples was parsed and dropped once already, which is +// what these lock down. +func TestResolveMaxSamples_Precedence(t *testing.T) { + withOptions := &project.Eval{MaxSamples: 25} + + assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") + assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, nil)) + assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") + + // Zero in config is absent, not a cap of zero: a cap of zero would send + // nothing at all. + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) +} + +// The level is the eval's alone. A per-run override would put two incomparable +// result sets under one eval's history, and would bypass the +// supported_evaluation_levels check `azd up` does against the declared level. +func TestResolveLevel_ComesFromTheEval(t *testing.T) { + declared := &project.Eval{ + EvaluationLevel: project.EvaluationLevelConversation, + } + + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel(declared)) + assert.Empty(t, resolveLevel(&project.Eval{}), "unset defers to the service default") + assert.Empty(t, resolveLevel(nil)) +} + +// A group's target decides which run-time fields its criteria can bind. Getting +// this wrong passes validation and then errors on every row. +func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { + assert.Nil(t, sampleBindingsFor("prompt"), + "an unrecognized target must bind nothing rather than guess at agent fields") +} + +// The level filter is what keeps a conversation evaluator from being sent turn +// fields and the reverse. Both directions matter. +func TestSelectLevelFields_KeepsOnlyTheLevelsShape(t *testing.T) { + accepted := []string{"query", "response", "messages", "tool_definitions"} + + conv := selectLevelFields(accepted, nil, project.EvaluationLevelConversation) + assert.Contains(t, conv, "messages") + assert.NotContains(t, conv, "query") + assert.NotContains(t, conv, "response") + assert.Contains(t, conv, "tool_definitions", "fields outside the split are untouched") + + turn := selectLevelFields(accepted, nil, project.EvaluationLevelTurn) + assert.Contains(t, turn, "query") + assert.Contains(t, turn, "response") + assert.NotContains(t, turn, "messages") + + // An evaluator offering only one shape is left alone, whatever the level. + only := []string{"query", "response"} + assert.Equal(t, only, selectLevelFields(only, nil, project.EvaluationLevelConversation)) + + // A required field is never dropped: a genuine conflict has to surface as a + // missing-field error rather than being reshaped away. + kept := selectLevelFields(accepted, []string{"query"}, project.EvaluationLevelConversation) + assert.Contains(t, kept, "query", "a required field survives the level filter") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go new file mode 100644 index 00000000000..d56b62c0bf6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What the previous run sent is history. A caller that logs it, or emits it +// under -o json, has to see what was recorded rather than what this run decided +// to send instead. +func TestPinReusedTraceWindow_DoesNotTouchWhatItWasGiven(t *testing.T) { + recorded := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", + AgentName: "support-agent", + StartTime: time.Now().Add(-24 * time.Hour).Unix(), + }, + } + before := *recorded.TraceSource + + pinned := pinReusedTraceWindow(recorded) + + require.NotSame(t, recorded, pinned) + assert.Equal(t, before, *recorded.TraceSource, "the recorded source is unchanged") + assert.NotZero(t, pinned.TraceSource.EndTime) +} + +// A window with no start says "everything", which is what it said when it was +// recorded. Closing it would freeze a declaration that never asked to be +// bounded, and each reattach would then grade a staler span than the last. +func TestPinReusedTraceWindow_LeavesAnUnboundedWindowUnbounded(t *testing.T) { + open := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", AgentName: "support-agent", + }, + } + + assert.Same(t, open, pinReusedTraceWindow(open)) + assert.Zero(t, open.TraceSource.EndTime) + assert.Zero(t, open.TraceSource.StartTime) +} + +// A recorded end early enough to put the start at or before the epoch would +// send a bound the wire drops, or a negative one, which is what a declaration +// is refused for. The length of the window is kept and reached back from now. +func TestPinReusedTraceWindow_KeepsAReattachedStartOutOfThePreEpoch(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + EndTime: 3600, + }) + + require.NotNil(t, ds.TraceSource) + assert.Positive(t, ds.TraceSource.StartTime) + assert.Equal(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go new file mode 100644 index 00000000000..1a015318e25 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/foundry/projectctx" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// NewRootCommand builds the `azd ai eval` command tree. +func NewRootCommand() *cobra.Command { + rootCmd, extCtx := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + Name: "eval", + Use: "eval [options]", + Short: fmt.Sprintf( + "Define and run Foundry evaluations from your terminal. %s", + color.YellowString("(Beta)"), + ), + }) + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + rootCmd.CompletionOptions.DisableDefaultCmd = true + + // The data-plane clients trace requests through the standard logger, which + // Go writes to stderr, so it has to be silenced unless debug was asked for. + // + // The SDK's own hook is chained rather than replaced, and cobra ignores + // PersistentPreRun entirely once PersistentPreRunE is set. The SDK sets + // cobra.EnableTraverseRunHooks, so this still runs alongside subcommand + // hooks. The cleanup func is discarded on purpose: log writes are + // unbuffered and the OS closes the file at exit. + sdkPreRun := rootCmd.PersistentPreRunE + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if sdkPreRun != nil { + if err := sdkPreRun(cmd, args); err != nil { + return err + } + } + // -e/--environment is parsed by the SDK into extCtx and then has to be + // acted on. Discarding extCtx left the flag accepted and ignored: + // `azd ai eval create -e staging` read the endpoint out of the default + // environment and wrote its eval id back there, and even a name azd + // itself rejects was accepted in silence. Set here rather than at each + // reader, so there is one answer to which environment this invocation + // is about. + cmd.SetContext(projectctx.WithSelectedEnvironment(cmd.Context(), extCtx.Environment)) + if err := projectctx.VerifySelectedEnvironment(cmd.Context()); err != nil { + return err + } + setupDebugLogging(cmd.Flags()) + return nil + } + + rootCmd.AddCommand( + newInitCommand(), + newDatasetCommand(), + newRunCommand(), + newEvaluatorCommand(), + newGenerateCommand(), + newJobCommand(), + newEvalCreateCommand(), + newEvalListCommand(), + newEvalShowCommand(), + newEvalDeleteCommand(), + newListenCommand(), + ) + + // The manifest declares the `metadata` capability, which azd uses to + // discover this extension's command tree. Without the command registered, + // that discovery fails with "unknown command". + rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.evaluations", func() *cobra.Command { + return rootCmd + })) + + return rootCmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go new file mode 100644 index 00000000000..778afa3d029 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -0,0 +1,1159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// Terminal run states reported by the service. +var terminalRunStates = map[string]bool{ + "completed": true, + "failed": true, + "canceled": true, + "cancelled": true, + "error": true, +} + +// runCompleted turns a run that did not complete into an error, so that a +// caller who waited for it exits non-zero. +// +// The results have already been printed by the time this is asked, which is +// the point: a run that errored has a reason worth reading, and reporting it +// and then exiting 0 tells a pipeline the evaluation passed. It is checked +// before the gate because the gate's exit code means "the evaluation +// regressed", and a run that never produced results has not regressed — it did +// not run. Distinguishing those two is what the separate code is for. +func runCompleted(run *eval_api.OpenAIEvalRun) error { + if run == nil { + return nil + } + switch strings.ToLower(run.Status) { + case "completed", "": + return nil + } + return messages.RunFinishedWithStatus(run.ID, run.Status) +} + +// runIsTerminal reports whether the run has stopped moving. +// +// A gate read from a run still in progress is read from partial counts: it can +// fail a run that would have passed, and it can pass one that has not finished +// failing. +// +// Derived from the polling vocabulary rather than repeating it: a state the +// poller stops waiting on is a state whose counts are final, and keeping two +// lists let "error" fall out of this one -- which told anyone gating an errored +// run to pass --wait, on a run that had already stopped. The empty status is +// the one deliberate difference: polling keeps waiting on a run the service has +// not described yet, while a gate reads the counts it was handed. +func runIsTerminal(run *eval_api.OpenAIEvalRun) bool { + if run == nil { + return false + } + status := strings.ToLower(run.Status) + return status == "" || terminalRunStates[status] +} + +// newRunCommand builds the run group. +// +// `run` is a group, not an executable verb: once `run output` exists, a bare +// `run` would make `azd ai eval run list` read as "run the thing called list". +func newRunCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "Start and inspect evaluation runs.", + } + addRunSubcommands(cmd) + cmd.AddCommand(buildRunCommand( + "start", "Start a run of an eval that has been deployed.")) + return cmd +} + +// buildRunCommand builds `run start`. +func buildRunCommand(use, short string) *cobra.Command { + var ( + groupName string + datasetName string + runName string + maxSamples int + wait bool + failOn string + endpointFlg string + evalPath string + ) + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + // Parsed before any network work, so a malformed threshold costs + // nothing to find out about. + threshold, err := parseGate(failOn) + if err != nil { + return err + } + // A gate is a verdict on a result. Returning before there is one + // used to drop the gate silently, so `--no-wait --fail-on ...` + // exited 0 however the run turned out -- a pipeline that believes + // it is gated and is not. + if !wait && threshold.set { + return messages.GateNeedsTheWait() + } + // resolveMaxSamples reads anything not above zero as "no cap", so a + // negative one sent the whole dataset to a billed run. + if maxSamples < 0 { + return messages.NegativeMaxSamplesFlag(maxSamples) + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // One flag takes a name or an id. A declared name also brings the + // declaration, which is what says where rows come from; a bare id + // has none, so the pairing comes from the eval's previous run. + evalDir, err := ec.evalDir(ctx, evalPath) + if err != nil { + return err + } + ref, err := ec.resolveEvalRef(ctx, evalDir, chooseEvalIn(cmd, evalDir, groupName)) + if err != nil { + return err + } + evalID := ref.ID + group := ref.Eval + configPath := ref.ConfigPath + + if datasetName != "" { + if !ref.Declared() { + return messages.DatasetOverrideNeedsDeclaredEval() + } + if _, ok := ref.Config.DatasetDeclaration(datasetName); !ok { + return messages.DatasetNotInCatalog( + datasetName, filepath.ToSlash(configPath)) + } + // The eval keeps its own declaration; only this run reads elsewhere. + overridden := *group + overridden.Dataset = datasetName + overridden.Source = nil + group = &overridden + } + + if ref.Declared() { + if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { + return err + } + } + + var dataSource *eval_api.EvalRunDataSource + switch { + case group == nil: + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + default: + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) + } + if err != nil { + return err + } + + if runName == "" { + base := "eval" + if group != nil { + base = group.Name + } + runName = fmt.Sprintf("%s-%s", base, time.Now().UTC().Format("20060102-150405")) + } + + metadata := map[string]string{} + if lvl := resolveLevel(group); lvl != "" { + metadata["evaluation_level"] = lvl + } + // The eval carries its name in its own metadata, but a run is read + // on its own, and an id is not what the author called it. + if group != nil && group.Name != "" { + metadata[metaEvalName] = group.Name + } + // Recorded per run, not read from the configuration at list time: + // comparing two runs is the point of that listing, and the dataset + // under an eval can change between them. A source-backed run scored + // no dataset, so it records none. + if group != nil && group.Dataset != "" && group.Source == nil { + metadata[metaDataset] = group.Dataset + if v := ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)); v != "" { + metadata[metaDatasetVersion] = v + } + } + + run, err := ec.evalClient.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: runName, + DataSource: dataSource, + Metadata: metadata, + }) + if err != nil { + return messages.StartingRun(err) + } + + // Remembered per group as well as globally: a single shared key + // belongs to whichever group ran last, so another group asking for + // "the last run" would be handed one that is not its own. + ec.remember(ctx, idKey("evalrun", evalID), run.ID) + if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { + // Persisting the run id is a convenience for later commands. + // Reported on stdout because azd does not surface an + // extension's stderr, and skipped outside a project. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprint(out, messages.Warning(err)) + } + } + + if !wait { + if isJSON(cmd) { + return emitJSON(out, startedRun(run, evalID, group)) + } + fmt.Fprint(out, messages.RunStarted(run.ID, run.Status)) + fmt.Fprint(out, messages.ReattachToRun(run.ID, evalID)) + return nil + } + + final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) + if errors.Is(err, errWaitBudgetSpent) { + // A gate asked for a verdict that never arrived. Exiting 0 here + // would tell a pipeline the gate passed, which is the silent + // drop --no-wait is refused for, reached by running long. + if threshold.set { + return messages.GateOutlivedTheWait(run.ID, waitBudget) + } + // The run did not fail, the wait ran out. Same contract as + // --no-wait: exit 0 and say how to pick it back up. + if isJSON(cmd) { + return emitJSON(out, startedRun(run, evalID, group)) + } + fmt.Fprint(out, messages.WaitBudgetSpent(run.ID, waitBudget)) + fmt.Fprint(out, messages.ReattachToRun(run.ID, evalID)) + return nil + } + if err != nil { + return err + } + final = ec.withPortalLink(ctx, evalID, final) + + if isJSON(cmd) { + if err := emitJSON(out, final); err != nil { + return err + } + } else if err := renderRun(out, final, ec.runMeans(ctx, evalID, final)); err != nil { + return err + } + + // Last, so that the results are reported whether or not the gate + // holds: a pipeline that only learns it failed is worse off than + // one that can see by how much. + if err := runCompleted(final); err != nil { + return err + } + applyGate(cmd, threshold, final) + return nil + }, + } + + cmd.Flags().StringVar(&groupName, "eval", "", + "Name of the eval to run, or its id. Defaults to the only one declared.") + cmd.Flags().StringVar(&datasetName, "dataset", "", + "Catalog dataset to read instead of the one the eval declares. "+ + "Must satisfy the eval's column schema.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + "Cap the rows sent from the dataset.") + cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + addFailOnFlag(cmd, &failOn) + // The spec documents --no-wait, and cobra does not derive it from a bool. + var noWait bool + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") + cmd.PreRun = func(*cobra.Command, []string) { + if noWait { + wait = false + } + } + cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + addEvalPathFlag(cmd, &evalPath) + + return cmd +} + +// checkDatasetRegistered fails when the group's local dataset has edits that +// were never deployed. +// +// A run sends a local dataset inline, so without this the run would evaluate +// content that no registered version corresponds to: the results are attributed +// to the eval but cannot be traced back to a dataset version, which +// makes them impossible to reproduce or compare. +// +// The check only applies once a deploy has recorded a fingerprint. Before that +// there is nothing to have drifted from, and running is how a group first comes +// into existence. +func (ec *evalContext) checkDatasetRegistered( + ctx context.Context, + cfg *project.EvalConfig, + group *project.Eval, + configPath string, +) error { + localPath := localDatasetPath(configPath, group) + if localPath == "" { + return nil + } + + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok { + return nil + } + + recorded := ec.getEnvValue(ctx, project.FingerprintKey("dataset", decl.Name)) + if recorded == "" { + return nil + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + // Reading the file is the run's problem to report, not this check's. + return nil + } + if digest == recorded { + return nil + } + + return messages.DatasetHasUnregisteredEdits(decl.Name, ec.deployCommand(ctx)) +} + +// reuseDataSourceFromLastRun rebuilds a run's data source from the group's most +// recent run. +// +// `--eval-id` deliberately ignores the config, but a run still needs a target +// and a dataset, and an eval carries neither: the group holds only its +// testing criteria, and the dataset travels on the run. The previous run is the +// only place that pairing survives, so re-running a group means repeating what +// it last ran. +func (ec *evalContext) reuseDataSourceFromLastRun( + ctx context.Context, + evalID string, +) (*eval_api.EvalRunDataSource, error) { + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + if eval_api.IsNotFound(err) { + // The eval itself is missing, which is worth saying plainly rather + // than as forty lines of the 404 that discovered it. + return nil, messages.EvalNotFound(evalID) + } + return nil, messages.ReadingPreviousRuns(evalID, err) + } + if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { + return nil, messages.EvalHasNoPreviousRun(evalID) + } + return pinReusedTraceWindow(list.Data[0].DataSource), nil +} + +// legacyTraceLookbackHours is the window a legacy source with no lookback ran +// under: the service's own default of seven days. +// +// Recorded here because the old data source had no start bound of its own -- +// it carried agent_name, lookback_hours, end_time and max_traces, and nothing +// else -- so a run that set no lookback was graded over whatever the service +// chose. Carrying such a run forward with no start at all would widen it to all +// of history instead. +const legacyTraceLookbackHours = 24 * 7 + +// pinReusedTraceWindow closes the window a reattached run repeats. +// +// A run reached by id repeats whatever data source the last one sent, and a +// trace window with a start and no end means "up to now". Replaying it a week +// later grades a week more than the run it was copied from, and the run after +// that more again, so the span grows without limit and nothing says so. +// +// A window with no start at all is repeated as it stands: it says "everything", +// which is what it said when it was recorded, and closing it would freeze a +// declaration that never asked to be bounded. So is a window that already has +// an end, which cannot widen and is not this function's to move. +// +// It is graded over the span it covers rather than the span it covered: the +// declaration is where a window that should move with each run comes from, and +// a run reached by id has no declaration to read. Freezing it once is the +// closest a shape with no lookback can come to one. +// +// Pinning the end at now also excludes traces the service has not finished +// ingesting, which an open end would have picked up on the next run. +// +// The argument is never modified: what the previous run sent is history, and a +// caller that logs or emits it should see what was recorded. +func pinReusedTraceWindow(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { + switch { + case ds == nil: + return ds + case ds.Type == eval_api.EvalRunDataSourceTypeTraces: + return upgradeLegacyTraceSource(ds) + case ds.Type == eval_api.EvalRunDataSourceTypeTracePreview: + if ds.TraceSource == nil || ds.TraceSource.EndTime != 0 || ds.TraceSource.StartTime == 0 { + return ds + } + pinned := *ds + filter := *ds.TraceSource + filter.EndTime = time.Now().Unix() + pinned.TraceSource = &filter + return &pinned + default: + return ds + } +} + +// upgradeLegacyTraceSource carries a run recorded under the old trace shape +// onto the one that keeps what it is given. +// +// Without it, an eval whose last run predates the change would keep sending the +// version-blind source for good, and nothing would say so. +func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { + // Without an agent the preview shape carries no filter at all, and + // omitempty drops it: the reattached run would read every agent's spans, + // a broader and costlier query than the one it is repeating. Repeating + // what was recorded is the lesser wrong. + if ds.AgentName == "" { + return ds + } + end := time.Now() + if ds.EndTime > 0 { + end = time.Unix(ds.EndTime, 0) + // A recorded end in the future would close the window after the last + // trace that exists, which reads nothing past now and says nothing. + if end.After(time.Now()) { + end = time.Now() + } + } + // The recorded values are whatever an older build sent, from before the + // bounds existed, so they are clamped rather than trusted: a lookback beyond + // what a window may cover reaches back further than any trace was recorded, + // and the reattached run reads nothing. + hours := ds.LookbackHours + if hours <= 0 || hours > project.MaxLookbackHours { + hours = legacyTraceLookbackHours + } + start := end.Add(-time.Duration(hours) * time.Hour) + // A recorded end early enough to put the start at or before the epoch would + // send a bound the wire drops, or a negative one -- the same silence a + // declaration is refused for. Reaching back from now instead keeps the + // length of the window the run asked for. + if start.Unix() <= 0 { + end = time.Now() + start = end.Add(-time.Duration(hours) * time.Hour) + } + // Only reachable on a machine whose clock is set before about 1980, where + // even now minus the longest window a declaration may name lands in the + // pre-epoch. Dropping the bound says "everything", which is at least what + // the legacy shape said when it carried no start. + if start.Unix() <= 0 { + start = time.Time{} + } + // A negative cap is no cap at all, and leaving it off means the service's + // own default of a thousand traces -- a bigger, costlier run than the one + // being repeated. The cap `init` writes is bounded and can be raised in the + // declaration, which is the only place a considered value can come from. + maxTraces := ds.MaxTraces + if maxTraces < 0 { + maxTraces = project.DefaultScaffoldMaxTraces + } + // The old shape carried no version, so this pins nothing that was not + // pinned before; it stops the service choosing differently run to run only + // once the declaration names one. + return eval_api.NewTracePreviewDataSource(ds.AgentName, "", start, end, maxTraces) +} + +// runnableEval refuses a declaration this run could not carry out. +// +// The rules live with the check the configuration runs, so the two cannot come +// to different conclusions about the same eval. Only the wrapper differs: the +// configuration has an index to name and a run does not. +func runnableEval(group *project.Eval) error { + if err := project.ValidateRunnable(group); err != nil { + return messages.InEval(group.Name, err) + } + return nil +} + +// buildRunDataSource binds the eval's rows to the run. +// +// Three shapes, in the order the configuration decides them. A `source:` block +// hands the gathering to the service and sends nothing local. Otherwise the +// rows come from a dataset, and `target:` says what to invoke for each one -- +// including nothing at all, when the rows already hold both sides. +// +// This is where a declaration is refused, not merely where it is read. Resolving +// an eval by name does not validate what it says about itself, and a run reached +// by id has no declaration to validate, so every contradiction the configuration +// names has to be answered here as well. Settling one by evaluation order sends +// a request that succeeds and grades something the file did not ask for. +func (ec *evalContext) buildRunDataSource( + ctx context.Context, + group *project.Eval, + configPath string, + maxSamples int, +) (*eval_api.EvalRunDataSource, error) { + if group == nil { + return nil, messages.NoEvalToRun() + } + if err := runnableEval(group); err != nil { + return nil, err + } + if group.Dataset != "" && configPath != "" && !datasetIsDeclared(configPath, group) { + return nil, messages.InEval(group.Name, messages.DatasetNotDeclared(group.Dataset)) + } + + if group.Source != nil { + switch group.Source.Type { + case project.SourceTypeTraces: + return tracesDataSource(group) + default: + return responsesDataSource(group) + } + } + + var ds *eval_api.EvalRunDataSource + switch { + case group.Target == nil || group.Target.Name == "": + // Nothing to invoke: the dataset is scored as it stands. + ds = eval_api.NewDatasetOnlyDataSource() + case group.Target.Type == project.TargetTypeModel: + ds = eval_api.NewModelTargetDataSource(group.Target.Name) + default: + ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + } + + if group.Dataset == "" { + return nil, messages.EvalHasNoDataset(group.Name) + } + + // A local source is read from disk; anything else is already registered and + // has to be fetched. Either way the rows are sent inline, because a run's + // file_id means an uploaded file and a dataset name is not one: sending the + // name is rejected with "invalid data source file ids". + localPath := localDatasetPath(configPath, group) + if localPath == "" { + items, err := ec.readRegisteredDataset(ctx, group.Dataset, maxSamples) + if err != nil { + return nil, err + } + ds.SetFileContent(items) + return ds, nil + } + + items, err := readJSONL(localPath, maxSamples) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, messages.DatasetFileEmpty(localPath) + } + ds.SetFileContent(items) + return ds, nil +} + +// tracesDataSource evaluates conversations the agent already had. +// +// The service reads them from Application Insights, so the agent has to be +// emitting gen_ai.input.messages / gen_ai.output.messages for anything to be +// found. `agent_name` filters the traces; it is not a target, because a trace +// run invokes nothing. +func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { + // runnableEval has already refused an empty one; read rather than assumed, + // because the agent name is what the whole request is about. + agent := project.TraceAgentName(group.Source, group.Target) + if agent == "" { + return nil, messages.InEval(group.Name, messages.TraceSourceNeedsAnAgent()) + } + + start, end, err := traceWindow(group.Name, group.Source) + if err != nil { + return nil, err + } + return eval_api.NewTracePreviewDataSource( + agent, + group.Source.AgentVersion, + start, + end, + group.Source.MaxTraces, + ), nil +} + +// traceWindow resolves the bounds of the span a trace run reads. +// +// The rules live in the project package, with the check the configuration runs, +// so a source is judged the same way whichever door the eval came through. +func traceWindow(evalName string, source *project.SourceDecl) (start, end time.Time, err error) { + start, end, err = project.ValidateSource(source) + if err != nil { + return time.Time{}, time.Time{}, messages.InEval(evalName, err) + } + return start, end, nil +} + +// responsesDataSource evaluates responses the project already stored. +func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { + if len(group.Source.ResponseIDs) == 0 { + return nil, messages.InEval(group.Name, messages.ResponsesSourceNeedsResponseIDs()) + } + return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil +} + +// readRegisteredDataset fetches a published dataset's rows, optionally keeping +// only the first n. +// +// The rows have to be fetched because a run cannot reference a dataset by +// name: `file_id` means an uploaded file, and passing a dataset name there is +// rejected. Fetching also makes --max-samples mean the same thing whether the +// dataset is local or published, which a file reference could not — that +// source carries no row limit. +func (ec *evalContext) readRegisteredDataset( + ctx context.Context, + name string, + maxSamples int, +) ([]map[string]any, error) { + version := ec.getEnvValue(ctx, versionKey("dataset", name)) + if version == "" { + versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return nil, messages.ReadingDataset(name, err) + } + if versions != nil { + version = dataset_api.LatestVersion(versions.Value) + } + } + if version == "" { + return nil, messages.DatasetHasNoVersionsToRead(name) + } + + content, err := ec.datasetClient.DownloadDatasetContent( + ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, messages.ReadingDatasetVersion(name, version, err) + } + + items, err := readJSONLBytes(content, maxSamples) + if err != nil { + return nil, messages.ReadingDatasetVersion(name, version, err) + } + if len(items) == 0 { + return nil, messages.DatasetVersionEmpty(name, version) + } + return items, nil +} + +// datasetColumnsFromPath reads one row to learn the dataset's shape. An empty +// path, or an unreadable file, yields nil. +func datasetColumnsFromPath(localPath string) map[string]bool { + if localPath == "" { + return nil + } + // One row is enough to learn the shape. + items, err := readJSONL(localPath, 1) + if err != nil || len(items) == 0 { + return nil + } + columns := make(map[string]bool, len(items[0])) + for name := range items[0] { + columns[name] = true + } + return columns +} + +// localDatasetPath resolves the dataset's local source relative to the config +// file, returning empty when the dataset is registered rather than local. +func localDatasetPath(configPath string, group *project.Eval) string { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil || group == nil { + return "" + } + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok || decl.Source == "" { + return "" + } + if filepath.IsAbs(decl.Source) { + return decl.Source + } + return filepath.Join(filepath.Dir(configPath), decl.Source) +} + +// datasetIsDeclared says whether the configuration's catalog holds the dataset +// this eval names. +// +// Without it a mistyped name falls through to a registry read and comes back as +// a 404 for a dataset nobody ever registered, which sends the reader to the +// service rather than to the line they mistyped. Answered yes when there is no +// configuration to ask: an eval reached by id has no catalog. +func datasetIsDeclared(configPath string, group *project.Eval) bool { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil || cfg == nil { + return true + } + _, ok := cfg.DatasetDeclaration(group.Dataset) + return ok +} + +// readJSONL reads newline-delimited JSON, optionally truncating to limit rows. +func readJSONL(path string, limit int) ([]map[string]any, error) { + f, err := os.Open(path) + if err != nil { + return nil, messages.ReadingDataset(path, err) + } + defer f.Close() + + items, err := scanJSONL(f, limit) + if err != nil { + return nil, messages.ReadingDataset(path, err) + } + return items, nil +} + +// readJSONLBytes parses JSONL already in memory, which is how a registered +// dataset arrives. +func readJSONLBytes(content []byte, limit int) ([]map[string]any, error) { + return scanJSONL(bytes.NewReader(content), limit) +} + +// scanJSONL reads rows until the limit is reached, so a subset costs only the +// rows it needs to parse. +func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { + var items []map[string]any + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + line := 0 + for scanner.Scan() { + line++ + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return nil, messages.JSONLLineInvalid(line, err) + } + items = append(items, row) + if limit > 0 && len(items) >= limit { + break + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return items, nil +} + +// resolveLevel prefers the flag, then the eval's own declaration. +// resolveLevel is the eval's declared scoring granularity. +// +// There is no per-run override: the level decides the row mapping, so two +// levels under one eval would put incomparable result sets in the same history, +// and it would bypass the supported_evaluation_levels check `azd up` does +// against the declared level. A second level is a second eval. +func resolveLevel(group *project.Eval) string { + if group != nil { + return group.EvaluationLevel + } + return "" +} + +// resolveMaxSamples prefers the flag, then the eval's own declaration, matching +// how the evaluation level resolves. +// +// Without this, max_samples parsed and did nothing: an eval that caps its +// sample count in config would send the whole dataset, and only a flag on every +// invocation would honour the cap. +func resolveMaxSamples(flag int, group *project.Eval) int { + if flag > 0 { + return flag + } + if group != nil && group.MaxSamples > 0 { + return group.MaxSamples + } + return 0 +} + +// errWaitBudgetSpent says the run outlived the wait, not that anything failed. +// +// It is handled where --no-wait is: the run is still going server-side, so the +// caller is handed the same reattach line and the same exit code. +var errWaitBudgetSpent = errors.New("wait budget spent") + +// waitBudget bounds a foreground wait. +// +// Not a policy about how long an evaluation may take -- it is a guard against +// waiting on a run that will never reach a terminal state. Runs are scored +// sequentially at roughly 40s a sample, so this clears a few hundred samples +// before it ever fires. +const waitBudget = 2 * time.Hour + +// pollRun waits for the run to reach a terminal state, reporting status changes. +func (ec *evalContext) pollRun( + ctx context.Context, + evalID, runID string, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (*eval_api.OpenAIEvalRun, error) { + const interval = 5 * time.Second + lastStatus := "" + deadline := time.Now().Add(waitBudget) + + for { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, messages.PollingRun(runID, err) + } + if run.Status != lastStatus { + lastStatus = run.Status + if !jsonMode { + fmt.Fprint(out, messages.RunStatusLine(run.Status)) + } + } + if terminalRunStates[strings.ToLower(run.Status)] { + return run, nil + } + if time.Now().After(deadline) { + return nil, errWaitBudgetSpent + } + select { + case <-ctx.Done(): + // Name what is still running, or the run is lost to whoever + // interrupted the wait. + return nil, messages.WaitInterrupted(runID, ctx.Err()) + case <-time.After(interval): + } + } +} + +// startedRunHandoff is what `run start --no-wait -o json` returns. +// +// It is a handoff rather than a dump of the service object. The pipeline that +// started the run has to come back for it later, and doing that needs exactly +// three things: the run, the eval it belongs to, and a name a human can read +// in the log that reports it. The service object carries none of the third and +// buries the first two under the data source, the metadata and every field the +// API happens to return, so a script reading it would depend on a shape this +// extension does not control. +type startedRunHandoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + EvalName string `json:"eval_name,omitempty"` + // Which rows the run scored. A pipeline that records only a pass rate + // cannot say later what the rate was measured against. + Dataset string `json:"dataset,omitempty"` + DatasetVersion string `json:"dataset_version,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt string `json:"created_at,omitempty"` +} + +// startedRun builds the handoff. +func startedRun( + run *eval_api.OpenAIEvalRun, + evalID string, + group *project.Eval, +) startedRunHandoff { + handoff := startedRunHandoff{ + RunID: run.ID, + EvalID: evalID, + Status: run.Status, + CreatedAt: timestampString(run.CreatedAt), + } + // Read back from the run rather than the configuration, so the handoff + // names what this run scored and not what the file says today. The create + // response does not always echo metadata, so the declaration is the + // fallback for the name. + handoff.Dataset = run.Metadata[metaDataset] + handoff.DatasetVersion = run.Metadata[metaDatasetVersion] + // Absent with --eval-id, where there is no config to take a name from. + if group != nil { + handoff.EvalName = group.Name + if handoff.Dataset == "" { + handoff.Dataset = group.Dataset + } + } + return handoff +} + +// timestampString renders a service timestamp as RFC 3339. +// +// The field arrives as epoch seconds on a run and as a formatted string +// elsewhere, so passing it through would hand a script a value whose type +// depends on which route produced it. +func timestampString(value any) string { + switch t := value.(type) { + case nil: + return "" + case string: + // Normalized, not passed through: the service returns sub-second + // precision and an offset here and epoch seconds elsewhere, so two + // listings would otherwise spell the same instant differently. + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC().Format(time.RFC3339) + } + return t + case float64: + return time.Unix(int64(t), 0).UTC().Format(time.RFC3339) + case int64: + return time.Unix(t, 0).UTC().Format(time.RFC3339) + case json.Number: + if seconds, err := t.Int64(); err == nil { + return time.Unix(seconds, 0).UTC().Format(time.RFC3339) + } + return t.String() + default: + return fmt.Sprint(value) + } +} + +// runMeans reads the run's rows to average each evaluator's score. +// +// Best effort: the summary is worth printing without the column, and a run +// that scored nothing has no rows to read. +func (ec *evalContext) runMeans( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) map[string]float64 { + if run == nil || run.ResultCounts == nil || run.ResultCounts.Total == 0 { + return nil + } + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil || items == nil { + return nil + } + return criteriaMeans(items.Data) +} + +// timestampTime reads a service timestamp, which arrives as epoch seconds on a +// run and as a formatted string elsewhere. +func timestampTime(value any) time.Time { + switch t := value.(type) { + case float64: + return time.Unix(int64(t), 0).UTC() + case int64: + return time.Unix(t, 0).UTC() + case string: + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC() + } + } + return time.Time{} +} + +// renderRun prints what a person needs after waiting for a run. +// +// means carries each criterion's average score, which the run summary does not +// return; it is nil when the rows were not fetched, and the column is dropped. +func renderRun( + out interface{ Write([]byte) (int, error) }, + run *eval_api.OpenAIEvalRun, + means map[string]float64, +) error { + fmt.Fprintln(out) + renderRunHeader(out, run) + + // A run that failed carries why, and it is usually the only actionable + // thing in the response — dropping it leaves the caller with just the word + // "failed". + if why := run.Failure(); why != "" { + fmt.Fprintf(out, "\n%s\n", why) + } + + renderCriteriaTable(out, run.PerTestingCriteria, means) + + // Counted over samples, not over verdicts: a sample that failed two + // evaluators is one sample to go and look at, and reporting it as two + // overstates how much is wrong. + if c := run.ResultCounts; c != nil && c.Total > 0 { + if rate, scored, ok := scoredPassRate(c); ok { + fmt.Fprint(out, messages.OverallPassRate( + fmt.Sprintf("%.1f%%", rate*100), c.Passed, scored, c.Total-scored)) + } + if c.Errored > 0 { + fmt.Fprint(out, messages.SamplesErrored(c.Errored)) + } + if c.Failed > 0 { + fmt.Fprint(out, messages.ViewFailingSamples()) + } + } + + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(out, messages.ReportLink(color.CyanString(url))) + } + return nil +} + +// renderRunHeader prints the run's identity above the per-evaluator table. +// +// The eval is named from the metadata the extension wrote at create time, +// because the run carries only an id and the id is not what anyone declared. +func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) { + fmt.Fprintf(out, "%-10s %s\n", "Run", run.ID) + if name := run.Metadata[metaEvalName]; name != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", name) + } else if run.EvalID != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) + } + if ds := runDatasetLine(run.Metadata); ds != "" { + fmt.Fprintf(out, "%-10s %s\n", "Dataset", ds) + } + fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) + } + if d := runDuration(run); d != "" { + fmt.Fprintf(out, "%-10s %s\n", "Duration", d) + } +} + +// runDuration reports how long the run took, or "" when either end is missing. +func runDuration(run *eval_api.OpenAIEvalRun) string { + start, end := timestampTime(run.CreatedAt), timestampTime(run.ModifiedAt) + if start.IsZero() || end.IsZero() || !end.After(start) { + return "" + } + d := end.Sub(start).Round(time.Second) + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + return fmt.Sprintf("%dm%02ds", int(d.Minutes()), int(d.Seconds())%60) +} + +// renderCriteriaTable prints one row per evaluator. +// +// Sorted by name so two runs of the same eval read the same way; the service +// returns the criteria in whatever order it evaluated them. +func renderCriteriaTable( + out interface{ Write([]byte) (int, error) }, + results []eval_api.EvalRunCriteriaResult, + means map[string]float64, +) { + if len(results) == 0 { + return + } + + sorted := append([]eval_api.EvalRunCriteriaResult(nil), results...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].TestingCriteria < sorted[j].TestingCriteria + }) + + width := len("EVALUATOR") + for _, r := range sorted { + if n := len(r.TestingCriteria); n > width { + width = n + } + } + + fmt.Fprintf(out, "\n%-*s %4s %4s %9s", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s\n", meanHeader(means)) + fmt.Fprintf(out, "%s %s %s %s%s\n", + strings.Repeat("-", width), "----", "----", "---------", meanRule(means)) + + for _, r := range sorted { + scored := r.Passed + r.Failed + fmt.Fprintf(out, "%-*s %4d %4d %9s", + width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + if means != nil { + if mean, ok := means[r.TestingCriteria]; ok { + fmt.Fprintf(out, " %10.1f", mean) + } else { + fmt.Fprintf(out, " %10s", "-") + } + } + fmt.Fprintln(out) + // Errors are not failures — the evaluator never reached a verdict — + // so they are named rather than folded into the fail column, where + // they would look like a quality problem. + if r.Errored > 0 { + fmt.Fprintf(out, "%-*s %s\n", width, "", errorNote(r.Errored)) + } + } +} + +// meanHeader and meanRule add the score column only when there are scores. +func meanHeader(means map[string]float64) string { + if means == nil { + return "" + } + return fmt.Sprintf(" %10s", "MEAN SCORE") +} + +func meanRule(means map[string]float64) string { + if means == nil { + return "" + } + return " " + strings.Repeat("-", 10) +} + +// criteriaMeans averages each evaluator's score over the rows it scored. +// +// The run summary reports pass and fail counts but no score, so a table that +// shows how close a passing evaluator came to failing has to read the rows. +// Errored and unscored rows are left out rather than counted as zero, which +// would drag the average toward a number no evaluator produced. +func criteriaMeans(items []eval_api.OutputItem) map[string]float64 { + sums := map[string]float64{} + counts := map[string]int{} + for _, item := range items { + for _, r := range item.Results { + if !r.Score.Defined() { + continue + } + name := r.Name + if name == "" { + name = r.Metric + } + sums[name] += float64(r.Score) + counts[name]++ + } + } + if len(counts) == 0 { + return nil + } + means := make(map[string]float64, len(counts)) + for name, n := range counts { + means[name] = sums[name] / float64(n) + } + return means +} + +// errorNote describes rows an evaluator could not score. +func errorNote(errored int) string { + return messages.ErroredNotScored(errored) +} + +// formatRate renders a share as a percentage, and a rate over nothing as a +// dash: 0.0%% would read as a total failure rather than as no data. +func formatRate(part, whole int) string { + if whole <= 0 { + return "-" + } + return fmt.Sprintf("%.1f%%", float64(part)/float64(whole)*100) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go new file mode 100644 index 00000000000..f4f8864aaf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Scenario 3 reads a trend off two rows of `run list`, and the dataset column +// is what makes the comparison honest: two rates over different rows are not +// the same claim. +func TestRunDataset_NameAndVersion(t *testing.T) { + got := runDataset(map[string]string{ + metaDataset: "support-agent-regression", + metaDatasetVersion: "1", + }) + + assert.Equal(t, "support-agent-regression (v1)", got) +} + +// A dataset recorded before it was published has a name but no version yet. +func TestRunDataset_NameWithoutVersion(t *testing.T) { + got := runDataset(map[string]string{metaDataset: "support-golden"}) + + assert.Equal(t, "support-golden", got) +} + +// Runs started before the extension recorded this show nothing. Falling back to +// the configuration would print today's dataset against a run that scored a +// different one, which is exactly the drift the column exists to reveal. +func TestRunDataset_UnrecordedShowsNothing(t *testing.T) { + assert.Empty(t, runDataset(nil)) + assert.Empty(t, runDataset(map[string]string{"evaluation_level": "turn"})) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go new file mode 100644 index 00000000000..76948cb9fb8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -0,0 +1,454 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeDataset drops a JSONL file beside a config that registers it, and +// returns the config path. An eval's dataset: is a catalog name, not a path, so +// the declaration is what makes the rows reachable. +func writeDataset(t *testing.T, rows string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "datasets"), 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "datasets", "d.jsonl"), []byte(rows), 0o600)) + configPath := filepath.Join(dir, "eval.yaml") + config := "datasets:\n - name: d\n source: ./datasets/d.jsonl\n" + require.NoError(t, os.WriteFile(configPath, []byte(config), 0o600)) + return configPath +} + +const oneRow = `{"query":"q","ground_truth":"a"}` + "\n" + +// A trace-backed eval is the first hero scenario, and it is the one shape that +// carries no dataset at all: the service gathers the rows itself. It used to be +// refused for "not naming a target agent", which is the whole point of it. +func TestBuildRunDataSource_Traces(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + MaxTraces: 500, + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + // The legacy azure_ai_traces shape discarded agent_version and start_time + // without saying so, and re-imposed its own lookback. + assert.Equal(t, eval_api.EvalRunDataSourceTypeTracePreview, ds.Type) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "agent_filter", ds.TraceSource.Type) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + assert.Equal(t, 500, ds.TraceSource.MaxTraces) + // lookback_hours is still honoured, as the window's start bound. Asserted + // as a distance from now, because a merely non-zero start is also what a + // lookback that reached forwards would produce. + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime, 60) + assert.Zero(t, ds.TraceSource.EndTime, "an open end means up to now") + // Nothing is invoked and nothing local is sent. + assert.Nil(t, ds.Target) + assert.Nil(t, ds.Source) +} + +// Pinning the version is the whole reason the preview shape is used: without +// it a redeployed agent is graded on whichever version the service picked. +func TestBuildRunDataSource_TracesPinsTheAgentVersion(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + AgentVersion: "2", + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "2", ds.TraceSource.AgentVersion) +} + +// An explicit window travels intact, in seconds and in UTC. The epochs are +// spelled out because a drift into local time, or into milliseconds, would +// still produce a window the service accepts and grades the wrong span of. +func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + StartTime: "2026-08-01T00:00:00Z", + EndTime: "2026-08-02T00:00:00Z", + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, int64(1785542400), ds.TraceSource.StartTime) + assert.Equal(t, int64(1785628800), ds.TraceSource.EndTime) +} + +// Every input the configuration refuses has to be refused here too. The two +// used to be separate checks and had drifted on six inputs, each accepted by +// one door and refused by the other, so which rules applied depended on how the +// eval was reached. +func TestBuildRunDataSource_TracesRefusesEverySourceTheConfigWould(t *testing.T) { + ec := &evalContext{} + build := func(source *project.SourceDecl) error { + source.Type = project.SourceTypeTraces + source.AgentName = "a" + _, err := ec.buildRunDataSource(context.Background(), + &project.Eval{Name: "trace-eval", Source: source}, "", 0) + return err + } + + cases := []struct { + name string + source project.SourceDecl + wantErr string + }{ + {"start that is not a time", project.SourceDecl{StartTime: "yesterday"}, "not a time"}, + {"start at year one", project.SourceDecl{StartTime: "0001-01-01T00:00:00Z"}, "traces were recorded at"}, + {"start at the unix epoch", project.SourceDecl{StartTime: "1970-01-01T00:00:00Z"}, "traces were recorded at"}, + {"negative lookback", project.SourceDecl{LookbackHours: -24}, "cannot be negative"}, + {"lookback past the bound", project.SourceDecl{LookbackHours: project.MaxLookbackHours + 1}, "beyond the"}, + {"negative cap", project.SourceDecl{MaxTraces: -5}, "source.max_traces"}, + { + "window declared twice over", + project.SourceDecl{StartTime: "2026-08-01T00:00:00Z", LookbackHours: 24}, + "keep one", + }, + { + "end before start", + project.SourceDecl{StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + "holds no traces", + }, + // Not a window rule. The run door used to accept and ignore these + // while the config refused them. + {"a turn cap traces do not read", project.SourceDecl{MaxTurns: 3}, "does not read"}, + { + "response ids traces do not read", + project.SourceDecl{ResponseIDs: []string{"resp_1"}}, + "does not read", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := build(&tc.source) + require.Error(t, err) + // The eval is named, not the agent: the reader has to know which + // entry to edit, and a file can declare several evals over one agent. + assert.Contains(t, err.Error(), `eval "trace-eval"`) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// The responses door runs the same check. It reads no window, so a window on it +// bounds nothing and only looks as though it does. +func TestBuildRunDataSource_ResponsesRefusesFieldsItDoesNotRead(t *testing.T) { + ec := &evalContext{} + + _, err := ec.buildRunDataSource(context.Background(), &project.Eval{ + Name: "responses-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeResponses, + ResponseIDs: []string{"resp_1"}, + // Present, so the missing-ids guard does not answer first. + LookbackHours: 24, + }, + }, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), `eval "responses-eval"`) + assert.Contains(t, err.Error(), "lookback_hours") + assert.Contains(t, err.Error(), "does not read") +} + +// agent_name under source: is a filter, but an eval that names a target and +// leaves the filter off still means "this agent's traces". +func TestBuildRunDataSource_TracesFallsBackToTargetName(t *testing.T) { + ec := &evalContext{} + + for _, target := range []*project.Target{ + {Type: project.TargetTypeAgent, Name: "support-agent"}, + // `target.type` is optional, and config validation accepts the + // fallback on the name alone, so whatever it accepts a run has to be + // able to send. The dataset branch reads an untyped target as an agent + // too; requiring the type here made a config that deployed cleanly fail + // every run. + {Name: "support-agent"}, + } { + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + Target: target, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + } +} + +// A model target names a deployment, not an agent. Filtering spans by a +// deployment name matches nothing, so the run would come back empty with no +// reason given; saying so is more use. +func TestBuildRunDataSource_TracesWillNotReadAModelTarget(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + Target: &project.Target{Type: project.TargetTypeModel, Name: "gpt-4o-mini"}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "agent_name") +} + +// With neither, the run cannot say whose conversations to read, and saying so +// is more use than letting the service return nothing. +func TestBuildRunDataSource_TracesWithoutAnAgentIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "source.agent_name") +} + +// Stored responses travel as rows carrying ids, with a data_mapping telling the +// service which field holds one. +func TestBuildRunDataSource_Responses(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "replay", + Source: &project.SourceDecl{ + Type: project.SourceTypeResponses, + ResponseIDs: []string{"resp_1", "resp_2"}, + MaxTurns: 3, + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) + require.NotNil(t, ds.ItemGenerationParams) + assert.Equal(t, 3, ds.ItemGenerationParams.MaxNumTurns) + assert.Equal(t, + map[string]string{"response_id": "{{item.response_id}}"}, + ds.ItemGenerationParams.DataMapping) + require.NotNil(t, ds.ItemGenerationParams.Source) + assert.Len(t, ds.ItemGenerationParams.Source.Content, 2) +} + +func TestBuildRunDataSource_ResponsesWithoutIDsIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "replay", + Source: &project.SourceDecl{Type: project.SourceTypeResponses}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "source.response_ids") +} + +// No target means the rows already hold both sides of the exchange, so the run +// scores them as they stand rather than invoking anything. +func TestBuildRunDataSource_NoTargetScoresTheDatasetAsItStands(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{Name: "recorded", Dataset: "d"} + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeJSONL, ds.Type) + assert.Nil(t, ds.Target) + require.NotNil(t, ds.Source) + assert.Len(t, ds.Source.Content, 1) +} + +// A model target was accepted by config validation and then sent as +// azure_ai_agent, so the run failed against a resource that does not exist. +func TestBuildRunDataSource_ModelTargetIsSentAsAModel(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{ + Name: "model-eval", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeModel, Name: "gpt-4o-mini"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_model", ds.Target.Type) + assert.Equal(t, "gpt-4o-mini", ds.Target.Model) + assert.Empty(t, ds.Target.Name, "a model target is addressed by deployment, not by agent name") +} + +func TestBuildRunDataSource_AgentTarget(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{ + Name: "agent-eval", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeAgentTarget, ds.Type) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_agent", ds.Target.Type) + assert.Equal(t, "support-agent", ds.Target.Name) +} + +// An eval with neither a dataset nor a source: has no rows from anywhere, and +// the error has to name both ways out rather than only the dataset. +func TestBuildRunDataSource_NoRowsFromAnywhere(t *testing.T) { + ec := &evalContext{} + + _, err := ec.buildRunDataSource(context.Background(), &project.Eval{Name: "empty"}, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "dataset:") + assert.Contains(t, err.Error(), "source:") +} + +// A misspelled source.type used to fall through to the dataset path, which +// scored the wrong rows and then blamed the eval for declaring no source. +// +// The run door is the first refusal for a declared eval as well as for one +// reached by id: resolving an eval by name checks only the name. +func TestBuildRunDataSource_UnknownSourceTypeIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "typo", + Source: &project.SourceDecl{Type: "trace"}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), `source.type "trace" is not supported`) + assert.NotContains(t, err.Error(), "references no dataset", + "a declared source must not be reported as no source at all") +} + +// The run door refuses every contradiction the configuration names. Resolving +// an eval by name does not check what it says about itself, and a run reached +// by id has no declaration to check, so settling one of these by evaluation +// order sends a request that succeeds and grades something else. +func TestBuildRunDataSource_RefusesADeclarationNoRunCouldCarryOut(t *testing.T) { + ec := &evalContext{} + + cases := []struct { + name string + eval project.Eval + wantErr string + }{ + { + "rows from two places", + project.Eval{ + Dataset: "d", + Source: &project.SourceDecl{Type: project.SourceTypeTraces, AgentName: "a"}, + }, + "declare one", + }, + { + // Read as "no cap", so the whole dataset went to a run billed per + // row when the file asked for fewer rows than that. + "a negative cap", + project.Eval{Dataset: "d", MaxSamples: -1}, + "max_samples cannot be negative", + }, + { + // Scored as though nothing were invoked, which is a different + // evaluation from the one that was written down. + "a target naming nothing", + project.Eval{Dataset: "d", Target: &project.Target{Type: project.TargetTypeAgent}}, + "target.name is required", + }, + { + "a target nothing can invoke", + project.Eval{Dataset: "d", Target: &project.Target{Type: "prompt", Name: "x"}}, + "is not supported", + }, + { + "a source that does not say what it reads", + project.Eval{Source: &project.SourceDecl{}}, + "source.type is required", + }, + { + "a responses source listing nothing", + project.Eval{Source: &project.SourceDecl{Type: project.SourceTypeResponses}}, + "source.response_ids is required", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.eval.Name = "e" + _, err := ec.buildRunDataSource(context.Background(), &tc.eval, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), `eval "e"`) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// --max-samples has to mean the same thing wherever the rows come from. +func TestBuildRunDataSource_MaxSamplesCapsLocalRows(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow+oneRow+oneRow) + group := &project.Eval{ + Name: "capped", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeAgent, Name: "a"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 2) + + require.NoError(t, err) + require.NotNil(t, ds.Source) + assert.Len(t, ds.Source.Content, 2) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go new file mode 100644 index 00000000000..2881096749c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A pipeline that starts a run with --no-wait has to come back for it. What it +// needs to do that is a fixed shape this extension controls, not whatever the +// API happened to return. +func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_01JQZX", + EvalID: "eval_ignored", + Status: "queued", + CreatedAt: "2026-07-31T21:04:11Z", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + DataSource: &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + }, + } + + raw, err := json.Marshal(startedRun(run, "eval_01JQZW", &project.Eval{Name: "support-agent-smoke"})) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + + assert.Equal(t, "evalrun_01JQZX", out["run_id"]) + assert.Equal(t, "support-agent-smoke", out["eval_name"]) + assert.Equal(t, "queued", out["status"]) + assert.Equal(t, "2026-07-31T21:04:11Z", out["created_at"]) + + // The eval the run was started against, which is the one the command + // resolved rather than whatever the run echoed back. + assert.Equal(t, "eval_01JQZW", out["eval_id"]) + + // Nothing the extension does not promise. A pipeline that could read the + // data source here would come to depend on it. + for _, leaked := range []string{"data_source", "metadata", "id", "report_url"} { + assert.NotContains(t, out, leaked, + "the handoff must not leak %q from the service object", leaked) + } +} + +// A script logging created_at should not have to know which route produced +// the run: the service sends epoch seconds here and a formatted string +// elsewhere, so the handoff settles on one. +func TestStartedRunNormalizesTheTimestamp(t *testing.T) { + for _, tc := range []struct { + name string + value any + want string + }{ + {"epoch seconds", float64(1785801525), "2026-08-03T23:58:45Z"}, + {"already formatted", "2026-07-31T21:04:11Z", "2026-07-31T21:04:11Z"}, + {"absent", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + handoff := startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", CreatedAt: tc.value}, "eval_1", nil) + assert.Equal(t, tc.want, handoff.CreatedAt) + }) + } +} + +// An empty one is omitted rather than reported as "", which a script would +// otherwise print as the eval's name. +func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { + raw, err := json.Marshal(startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "queued"}, "eval_1", nil)) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + assert.NotContains(t, out, "eval_name") + assert.Equal(t, "eval_1", out["eval_id"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go new file mode 100644 index 00000000000..86330aee194 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Scenario 3 answers "did my change help?" by reading two rows of `run list`, +// which only works if a row carries when it ran and how it scored. The columns +// were RUN ID / NAME / STATUS / RESULTS, so the question the scenario exists to +// answer could not be. +func TestRunListColumnsMatchTheScenario(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 15, Passed: 14, Failed: 1} + + assert.Equal(t, "15", sampleCount(counts), + "a rate over 15 samples and one over 200 are not the same claim") + assert.Equal(t, "93.3%", runPassRate(counts), + "the scenario compares 80.0% against 93.3%, so the row has to carry the rate") +} + +// The rate is the gate's arithmetic: passed over the rows that were scored, +// with errored and skipped outside it. A row a reader gates on must not +// disagree with the gate that acts on it. +// +// The list is the one view that shows a rate next to a sample count, so it also +// carries how many rows the rate covers. Without that, two passes and one +// errored row read as SAMPLES 3, PASS RATE 100.0%. +func TestRunListPassRateAgreesWithTheGate(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 4, Passed: 2, Failed: 1, Errored: 1} + + assert.Equal(t, "66.7% (3 scored)", runPassRate(counts), + "2 of the 3 rows that were scored passed, here and in the gate") + + g, err := parseGate("pass-rate=0.8") + assert.NoError(t, err) + assert.NotEmpty(t, g.breach(counts), + "the same counts that read 66.7% must breach an 80% threshold") + + // The errored row is outside the rate rather than counted as a failure, and + // the cell says so rather than reading as a clean sweep of the run. + assert.Equal(t, "100.0% (2 scored)", + runPassRate(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "nothing graded the errored row, so it is not a miss, but the rate is not the whole run") + + // Nothing unscored, nothing to qualify. + assert.Equal(t, "75.0%", + runPassRate(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1}), + "every row was scored, so the bare rate is the whole story") +} + +// A run that has not scored yet has no rate to show. An empty cell says that; +// "0.0%" would say the run failed. +func TestRunListOmitsARateItCannotCompute(t *testing.T) { + assert.Empty(t, runPassRate(nil)) + assert.Empty(t, runPassRate(&eval_api.EvalRunResultCounts{})) + assert.Empty(t, sampleCount(nil)) +} + +// Timestamps are RFC3339 in UTC, whichever shape the service sent. The service +// answers with epoch seconds on some routes and a string on others, and a list +// that renders both would not sort. +func TestRunListTimestampsAreRFC3339UTC(t *testing.T) { + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString(float64(1785575722))) + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString(int64(1785575722))) + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString("2026-08-01T09:15:22Z")) + assert.Empty(t, timestampString(nil)) +} + +// The table shows one rate per run because a column per evaluator stops being +// readable as soon as two runs score different evaluators. That makes `-o json` +// the only place a per-evaluator breakdown can be read, and the service does +// return it on the list route, so the runs go out unprojected. +// +// This pins the field name and that it survives marshalling. It does not catch +// someone replacing the emitted type with a projection, which is the way this +// would actually be lost -- that needs the command harness the reconciler tests +// now have. +func TestRunListJSONCarriesThePerEvaluatorBreakdown(t *testing.T) { + var buf bytes.Buffer + runs := []eval_api.OpenAIEvalRun{{ + ID: "evalrun_1", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 14, Failed: 1}, + }, + }} + + require.NoError(t, emitJSONList(&buf, runs)) + + assert.Contains(t, buf.String(), `"per_testing_criteria_results"`, + "the only place a script can read a per-evaluator result") + assert.Contains(t, buf.String(), "task_adherence") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go new file mode 100644 index 00000000000..e8916d89fc5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -0,0 +1,413 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "strconv" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// addRunSubcommands attaches the atomic run operations. +// +// `azd ai eval run` is a group rather than a verb; these are the operations it +// groups, each reachable without the config file. +func addRunSubcommands(cmd *cobra.Command) { + cmd.AddCommand( + newRunListCommand(), + newRunShowCommand(), + newRunCancelCommand(), + newRunDeleteCommand(), + newRunOutputCommand(), + ) +} + +func newRunListCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + limit int + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List runs for an eval.", + Long: "List runs for an eval.\n\n" + + "The table carries one pass rate per run. A per-evaluator breakdown " + + "cannot fit a column each and stay readable when runs score different " + + "evaluators, so `-o json` carries it instead, under " + + "`per_testing_criteria_results` on every run.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvalNotDeployed(evalID, ec.deployCommand(ctx)) + } + return messages.ListingRuns(evalID, err) + } + if isJSON(cmd) { + // Emitted whole, unlike `run start`, which hands back a small + // handoff. The table cannot show a per-evaluator breakdown, so + // this is the only place a script can read one; narrowing these + // to the table's columns would drop it silently. + var runs []eval_api.OpenAIEvalRun + if list != nil { + runs = list.Data + } + return emitJSONList(cmd.OutOrStdout(), runs) + } + if list == nil || len(list.Data) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.EvalHasNoRunsLine(evalID)) + return nil + } + + rows := make([][]string, 0, len(list.Data)) + for _, run := range list.Data { + rows = append(rows, []string{ + run.ID, + runDataset(run.Metadata), + timestampString(run.CreatedAt), + run.Status, + sampleCount(run.ResultCounts), + runPassRate(run.ResultCounts), + }) + } + return emitTable(cmd.OutOrStdout(), + []string{"RUN", "DATASET", "STARTED", "STATUS", "SAMPLES", "PASS RATE"}, rows) + }, + } + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().IntVar(&limit, "limit", 0, + "Return at most this many runs. Omit for the service default.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunShowCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + wait bool + failOn string + ) + + cmd := &cobra.Command{ + Use: "show [run]", + Short: "Show a single run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + threshold, err := parseGate(failOn) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + run = ec.withPortalLink(ctx, evalID, run) + + // Reattaching to a run started asynchronously: the pipeline that + // gates on it is often not the one that started it. + // + // Only a caller that waited is told a bad status through the exit + // code. Without --wait this is an inspection command: it was asked + // what happened, and answering that is a success whatever the + // answer. + gateOnStatus := wait + if wait { + run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if err != nil { + return err + } + } + + // The spec puts --fail-on on the commands that wait. Gating a run + // that is still moving would read partial counts; ignoring the flag + // would leave a pipeline believing it is gated when it is not. + if threshold.set && !runIsTerminal(run) { + return messages.GateNeedsATerminalRun(run.ID, run.Status) + } + + if isJSON(cmd) { + if err := emitJSON(cmd.OutOrStdout(), run); err != nil { + return err + } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } + applyGate(cmd, threshold, run) + return nil + } + + out := cmd.OutOrStdout() + fmt.Fprint(out, messages.RunHeading(run.ID)) + fmt.Fprint(out, messages.RunNameLine(run.Name)) + fmt.Fprint(out, messages.RunStatusDetail(run.Status)) + if counts := summarizeCounts(run.ResultCounts); counts != "" { + fmt.Fprint(out, messages.RunResultsLine(counts)) + } + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(out, messages.RunReportLine(color.CyanString(url))) + } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } + applyGate(cmd, threshold, run) + return nil + }, + } + cmd.Flags().BoolVar(&wait, "wait", false, + "Block until the run reaches a terminal state before reporting.") + addFailOnFlag(cmd, &failOn) + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// firstArg returns the positional argument, or empty when none was given. +func firstArg(args []string) string { + if len(args) > 0 { + return args[0] + } + return "" +} + +func newRunCancelCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "cancel [run]", + Short: "Cancel an in-flight run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + target, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + // Cancelling a run that already finished is a no-op worth naming, + // since the service reports success either way. Lowercased to match + // the polling path: the service's casing is not guaranteed. + if terminalRunStates[strings.ToLower(target.Status)] { + return messages.RunAlreadyFinished(target.ID, target.Status) + } + + canceled, err := ec.evalClient.CancelOpenAIEvalRun(ctx, evalID, target.ID) + if err != nil { + return messages.CancellingRun(target.ID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + status := canceled.Status + if status == "" { + status = "cancelling" + } + fmt.Fprint(cmd.OutOrStdout(), messages.RunIsNow(target.ID, status)) + return nil + }, + } + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunDeleteCommand removes a run. +// +// Runs accumulate — every `run start` adds one — and a run that evaluated the +// wrong dataset or target is noise in every later listing. The run is required +// rather than defaulted to the most recent, because deleting is not undoable +// and "the latest one" is a poor thing to guess at. +func newRunDeleteCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a run.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + runID := args[0] + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + if err := ec.evalClient.DeleteOpenAIEvalRun(ctx, evalID, runID); err != nil { + if eval_api.IsNotFound(err) { + return messages.RunNotFound(runID, evalID) + } + return messages.DeletingRun(runID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": runID, "eval_id": evalID, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.RunDeleted(runID)) + return nil + }, + } + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return messages.CountsSummary(counts.Passed, counts.Failed, counts.Errored) +} + +// metaDataset and metaDatasetVersion record which rows a run scored. The run's +// own data source cannot answer it: the rows travel inline, so the name that +// selected them is not in the request the service keeps. +const ( + metaDataset = "azd_dataset" + metaDatasetVersion = "azd_dataset_version" + // metaEvalName is the eval's declared name, recorded on the run because a + // run is read on its own and an id is not what the author called it. + metaEvalName = "azd_eval" + // metaAgent is the agent an eval targets. + metaAgent = "azd_agent" + // metaDescription carries an eval's description: the create request has no + // field of its own for it. + metaDescription = "azd_description" +) + +// runDataset renders the dataset a run scored, versioned when a version was +// recorded with it. +// +// A run started before this was recorded shows nothing rather than the name in +// the configuration today, which is the one thing the column exists to detect +// having changed. +func runDataset(metadata map[string]string) string { + name := metadata[metaDataset] + if name == "" { + return "" + } + if version := metadata[metaDatasetVersion]; version != "" { + return fmt.Sprintf("%s (v%s)", name, version) + } + return name +} + +// runDatasetLine is the same fact spelled for a detail view, where there is +// room for the whole word. +func runDatasetLine(metadata map[string]string) string { + name := metadata[metaDataset] + if name == "" { + return "" + } + if version := metadata[metaDatasetVersion]; version != "" { + return fmt.Sprintf("%s (version %s)", name, version) + } + return name +} + +// sampleCount is how many rows the run scored, which is what makes two rows of +// `run list` comparable: a rate over 15 samples and one over 200 are not the +// same claim. +func sampleCount(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return strconv.Itoa(counts.Total) +} + +// runPassRate is the same scored pass rate the gate uses, so a row a reader +// gates on cannot disagree with the gate. +// +// The rate is followed by the rows it was measured over whenever that is fewer +// than the run's samples. Without it the comparison view reads a run of two +// passes and one errored row as SAMPLES 3, PASS RATE 100.0%, which is the one +// place the scored denominator was not stated and so the one place a partly +// errored run looked perfect. +func runPassRate(counts *eval_api.EvalRunResultCounts) string { + rate, scored, ok := scoredPassRate(counts) + if !ok { + return "" + } + out := fmt.Sprintf("%.1f%%", rate*100) + if counts.Total > scored { + out += fmt.Sprintf(" (%d scored)", scored) + } + return out +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go new file mode 100644 index 00000000000..8259bfd6b35 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +func TestSummarizeCounts(t *testing.T) { + require.Equal(t, "", summarizeCounts(nil)) + require.Equal(t, "3 passed, 1 failed, 0 errored", + summarizeCounts(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1})) +} + +// Cancelling a finished run is rejected locally. The service reports success +// either way, so without this the CLI would claim it cancelled a run that had +// already completed. +func TestTerminalRunStatesCoverServiceVocabulary(t *testing.T) { + for _, status := range []string{"completed", "failed", "canceled", "cancelled", "error"} { + require.True(t, terminalRunStates[status], "%q should be terminal", status) + } + for _, status := range []string{"in_progress", "queued", "running", ""} { + require.False(t, terminalRunStates[status], "%q should not be terminal", status) + } +} + +// The atomic run operations have to be reachable as subcommands; the spec +// requires start, list, show and cancel to exist alongside the composite. +func TestRunCommandExposesAtomicSubcommands(t *testing.T) { + cmd := newRunCommand() + + found := map[string]bool{} + for _, sub := range cmd.Commands() { + found[sub.Name()] = true + } + for _, name := range []string{"start", "list", "show", "cancel"} { + require.True(t, found[name], "run should expose the %q subcommand", name) + } +} + +// `run start` is the atomic form of the composite and must accept the same +// flags, otherwise the two forms diverge. +func TestRunStartMirrorsCompositeFlags(t *testing.T) { + composite := newRunCommand() + + var start *cobra.Command + for _, sub := range composite.Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + + for _, flag := range []string{"eval", "dataset", "name", "max-samples", "wait", "no-wait"} { + require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) + } + + // The level decides the row mapping, so a per-run override would put two + // incomparable result sets under one eval. A second level is a second eval. + require.Nil(t, start.Flags().Lookup("level"), "run start must not offer --level") +} + +// Every command that acts on an eval says which one the same way. One flag +// takes a name from the configuration or a raw service id: an eval created +// outside a project has no declaration to name, and a second --eval-id beside +// it was accepted and silently ignored. +func TestEvalCommandsTakeOneEvalFlag(t *testing.T) { + subs := map[string]*cobra.Command{} + for _, sub := range newRunCommand().Commands() { + subs["run "+sub.Name()] = sub + if sub.Name() == "output" { + for _, leaf := range sub.Commands() { + subs["run output "+leaf.Name()] = leaf + } + } + } + + for _, name := range []string{ + "run list", "run show", "run cancel", + "run output list", "run output show", "run output export", + } { + cmd := subs[name] + require.NotNil(t, cmd, "%s should exist", name) + require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) + require.Nil(t, cmd.Flags().Lookup("eval-id"), + "%s must not keep --eval-id beside --eval", name) + } +} + +// --no-wait is documented in the spec, and cobra does not derive it from the +// --wait bool. It belongs to `run start`: `run` itself is a group. +func TestRunCommandAcceptsNoWait(t *testing.T) { + var start *cobra.Command + for _, sub := range newRunCommand().Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + require.NotNil(t, start.Flags().Lookup("no-wait"), "run start should accept --no-wait") + require.NotNil(t, start.Flags().Lookup("wait"), "run start should keep --wait") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go new file mode 100644 index 00000000000..0eb2bb36186 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -0,0 +1,701 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// newRunOutputCommand groups the per-sample views of a run. +// +// `run show` is the summary - how many passed. These are the rows: which ones +// failed, and why. +func newRunOutputCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "output", + Short: "Inspect the per-sample results of a run.", + } + cmd.AddCommand( + newRunOutputListCommand(), + newRunOutputShowCommand(), + newRunOutputExportCommand(), + ) + return cmd +} + +func newRunOutputListCommand() *cobra.Command { + var ( + failedOnly bool + outFile string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "list [run]", + Short: "List the per-sample results of a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + // The run carries totals and a per-criterion breakdown. The output + // items are the rows themselves, which is what "which one failed, + // and why" needs. A run that never produced any still renders its + // totals rather than failing. + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil { + return messages.ReadingRunResults(run.ID, err) + } + rows := items.Data + if failedOnly { + kept := make([]eval_api.OutputItem, 0, len(rows)) + for _, it := range rows { + if it.Failed() { + kept = append(kept, it) + } + } + rows = kept + } + + // A bare array, as every other list emits. Wrapping the rows beside + // the run made `-o json` the one listing a script could not iterate, + // and it failed silently: the loop walked the two keys instead. The + // run itself is what `run show` answers. + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return messages.Creating(outFile, err) + } + if err := emitJSONList(f, rows); err != nil { + _ = f.Close() + return err + } + // The last write is flushed by Close, so discarding its error + // reports success over a file that stops mid-row. + if err := f.Close(); err != nil { + return messages.Writing(outFile, err) + } + return nil + } + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), rows) + } + return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) + }, + } + + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunOutputShowCommand reads one evaluated row by its id. +// +// The listing truncates the input and the reason to keep a table readable, so +// this is how the whole of either is seen. +func newRunOutputShowCommand() *cobra.Command { + var ( + runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a single evaluated row.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + itemID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + item, err := ec.evalClient.GetOutputItem(ctx, evalID, run.ID, itemID) + if err != nil { + if eval_api.IsNotFound(err) { + return messages.OutputItemNotFound(itemID, run.ID) + } + return messages.ReadingOutputItem(itemID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), item) + } + return renderOutputItem(cmd.OutOrStdout(), item) + }, + } + + cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// writeExport renders a run in the requested export format. +// +// Separate from the command so the file path can close explicitly and report +// the failure. A deferred Close cannot reach an unnamed return, so discarding +// it exits 0 over an export that stops mid-row. +func writeExport(w io.Writer, format string, run *eval_api.OpenAIEvalRun) error { + switch format { + case formatCSV: + return writeResultsCSV(w, run) + case formatJSON: + return emitJSON(w, run) + case formatJSONL: + return writeResultsJSONL(w, run) + default: + return messages.ExportFormatUnsupported( + format, formatCSV, formatJSON, formatJSONL) + } +} + +func newRunOutputExportCommand() *cobra.Command { + var ( + format string + outFile string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "export [run]", + Short: "Export run results as CSV, JSON or JSONL.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + format = strings.ToLower(format) + // Checked against the same set the writer switches on: this guard + // used to name only json and csv, so --format jsonl was refused by a + // CLI whose own help offered it. + switch format { + case formatCSV, formatJSON, formatJSONL: + default: + return messages.ExportFormatUnsupported( + format, formatCSV, formatJSON, formatJSONL) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + if outFile == "" { + return writeExport(cmd.OutOrStdout(), format, run) + } + + f, createErr := os.Create(outFile) + if createErr != nil { + return messages.Creating(outFile, createErr) + } + writeErr := writeExport(f, format, run) + // The last write is flushed by Close, so discarding its error + // reports success over a file that stops mid-row. + closeErr := f.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return messages.Writing(outFile, closeErr) + } + return nil + }, + } + + cmd.Flags().StringVar(&format, "format", formatCSV, + fmt.Sprintf("Output format: %s, %s or %s.", formatCSV, formatJSON, formatJSONL)) + cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") + addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just `run start`. + addEvalPathFlag(cmd, new(string)) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalID resolves the eval a run command is about, from --eval or from +// the declaration the configuration holds. +// +// --eval accepts a name or a raw id on the one flag: an eval created outside a +// project has no declaration to name, and the environment records one id per +// name, so editing a declaration leaves every run of the previous eval +// reachable only by id. +// +// It takes no positional argument, deliberately. The positional on `run show`, +// `run cancel` and `run output *` is a *run* id, and a signature that accepted +// either would let one be resolved as the other -- a destructive verb aimed at +// a resource picked by accident. +// +// It reads no EVAL_ID either. Every deploy writes that key, so nothing tells a +// value meant for this declaration from one left behind by the eval it +// replaced; `run cancel` used to cancel a run of an eval the file no longer +// described. The declaration is asked instead, which is how `run start` +// decides it, so the two doors cannot pick different evals. +func resolveEvalID(cmd *cobra.Command, ec *evalContext, groupName string) (string, error) { + evalDir, err := ec.evalDir(cmd.Context(), evalPathFlag(cmd)) + if err != nil { + return "", err + } + // The same prompt `run start` gets. Without it a project declaring two + // evals could start a run by answering a question, and then not list, + // show or cancel it without repeating the answer as a flag. + ref, err := ec.resolveEvalRef(cmd.Context(), evalDir, chooseEvalIn(cmd, evalDir, groupName)) + if err != nil { + return "", err + } + return ref.ID, nil +} + +// addEvalFlag registers the flag that says which eval a command acts on. It +// takes a name from the configuration or a raw service id, which is why there +// is no second --eval-id beside it. +func addEvalFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", + "Name of the eval declared in the configuration, or its id.") +} + +// addEvalPathFlag registers --path on a command that reads the configuration. +// +// It defaults to empty rather than to ./evals so that "not given" stays +// distinguishable from "given the default", which is what lets the path `init` +// recorded take effect in between. +func addEvalPathFlag(cmd *cobra.Command, target *string) { + // No backticks: pflag reads a word in back quotes as the value placeholder, + // so `init` rendered the flag as "--path init" instead of "--path string". + cmd.Flags().StringVar(target, "path", "", + "Directory holding azure.eval.yaml. Defaults to the path init used, then ./evals.") +} + +// evalPathFlag reads --path from whichever command is resolving a declared +// name, so every one of them can be told where the configuration is. +// +// Read off the command rather than threaded through seven call sites. Without +// it only `run start` offered the flag, so a configuration outside ./evals +// could be run and then not listed, shown or cancelled -- the fallback that +// covers the difference is a path recorded in the azd environment, which a +// --project-endpoint caller does not have. +func evalPathFlag(cmd *cobra.Command) string { + if f := cmd.Flags().Lookup("path"); f != nil { + return f.Value.String() + } + return "" +} + +// latestOrNamedRun returns the named run, or the most recent one for the eval. +// +// explicit says whether the caller named the run rather than leaving it to +// default. A remembered run that no longer resolves is worth falling through +// on; one that was asked for by name is not. +// +// The remembered id is preferred over the service's listing, and deliberately. +// Listing looks like the fix for two concurrent starts leaving this key holding +// whichever wrote last, but ListOpenAIEvalRuns sends no order parameter, so +// "the first row" is not promised to be the newest; and `run cancel` defaults +// through here, so guessing would cancel a run this environment never started. +// The remembered id is at least scoped to the environment that made it. +func (ec *evalContext) latestOrNamedRun( + cmd *cobra.Command, + evalID, runID string, + explicit bool, +) (*eval_api.OpenAIEvalRun, error) { + ctx := cmd.Context() + + // The remembered run is per group. A single shared one belongs to whichever + // group ran last, and asking another group for it returns 404 rather than + // that group's own latest run. + if runID == "" { + runID = ec.getEnvValue(ctx, idKey("evalrun", evalID)) + } + if runID != "" { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err == nil { + return run, nil + } + if explicit { + return nil, messages.ReadingRun(runID, err) + } + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + if eval_api.IsNotFound(err) { + return nil, messages.EvalNotDeployed(evalID, ec.deployCommand(ctx)) + } + return nil, messages.ListingRuns(evalID, err) + } + if list == nil || len(list.Data) == 0 { + return nil, messages.EvalHasNoRuns(evalID) + } + return &list.Data[0], nil +} + +// renderOutputItem is the detail view for one evaluated row. +// +// This was the one `show` that emitted raw JSON whatever was asked for, which +// made the command a person reaches for after a failing listing the hardest one +// to read. The listing truncates the reason to a cell; this is where the whole +// of it lives, so the reasons are printed in full rather than wrapped or cut. +// +// Results are grouped by evaluator: a rubric reports one result per dimension, +// all carrying the evaluator's name, and printing them flat would read as +// several evaluators that happen to share a name. +func renderOutputItem(w io.Writer, item *eval_api.OutputItem) error { + if item == nil { + return messages.OutputItemEmpty() + } + if err := emitDetail(w, []field{ + {"Item", item.ID}, + {"Run", item.RunID}, + {"Status", item.Status}, + }); err != nil { + return err + } + + order := make([]string, 0, len(item.Results)) + byName := make(map[string][]eval_api.OutputResult, len(item.Results)) + for _, r := range item.Results { + if _, seen := byName[r.Name]; !seen { + order = append(order, r.Name) + } + byName[r.Name] = append(byName[r.Name], r) + } + + for _, name := range order { + results := byName[name] + fmt.Fprintln(w) + + // The service repeats the evaluator's name in `metric` for a + // single-score evaluator, so a group is only worth nesting when its + // results name dimensions of their own. + if len(results) == 1 && (results[0].Metric == "" || results[0].Metric == name) { + r := results[0] + fmt.Fprint(w, messages.OutputItemVerdict( + name, formatScore(r.Score), verdictWord(r))) + if r.Reason != "" { + fmt.Fprint(w, messages.OutputItemReason(r.Reason)) + } + continue + } + + fmt.Fprint(w, messages.OutputItemEvaluator(name)) + for _, r := range results { + label := r.Metric + if label == "" { + label = r.Name + } + fmt.Fprint(w, messages.OutputItemMetric( + label, formatScore(r.Score), verdictWord(r))) + if r.Reason != "" { + fmt.Fprint(w, messages.OutputItemReason(r.Reason)) + } + } + } + return nil +} + +// verdictWord spells a boolean the way the rest of the output does. +func verdictWord(r eval_api.OutputResult) string { + if !r.Judged() { + // The evaluator returned no verdict, which is not the same as returning + // a failing one -- it says nothing about the sample. + return "no verdict" + } + if r.DidPass() { + return "pass" + } + return "fail" +} + +// formatScore prints a judge's score at the two decimals the scale carries. +// formatScore shows a score, or a dash where there is none. An evaluator that +// errored on a row still sends a result, and its score decodes to NaN; printing +// that verbatim put "NaN" in the SCORE column. +func formatScore(score eval_api.LenientFloat) string { + if !score.Defined() { + return "-" + } + return strconv.FormatFloat(float64(score), 'f', 2, 64) +} + +// meanScoreOf averages a sample's scores so the list can tell a bare pass from +// a strong one. Pass/fail alone sent anyone asking "how well?" to the portal. +// +// Rows an evaluator errored on are left out rather than counted, the same rule +// criteriaMeans applies to the summary: averaging a NaN in makes the whole +// sample read NaN, and counting it as zero drags the mean toward a number no +// evaluator produced. +func meanScoreOf(results []eval_api.OutputResult) string { + total := 0.0 + scored := 0 + for _, r := range results { + if !r.Score.Defined() { + continue + } + total += float64(r.Score) + scored++ + } + if scored == 0 { + return "-" + } + return strconv.FormatFloat(total/float64(scored), 'f', 2, 64) +} + +func renderResults( + w io.Writer, + run *eval_api.OpenAIEvalRun, + items []eval_api.OutputItem, + failedOnly bool, +) error { + fmt.Fprint(w, messages.RunStatusHeading(run.ID, run.Status)) + + if c := run.ResultCounts; c != nil { + fmt.Fprint(w, messages.ResultTotals(c.Passed, c.Failed, c.Errored)) + } + + if len(run.PerTestingCriteria) > 0 { + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) > 0 { + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + } + } + + // The rows are the point of `results show`: totals say how many failed, + // these say which and why. + if len(items) == 0 { + if failedOnly { + fmt.Fprint(w, messages.NoFailingRows()) + } else { + fmt.Fprint(w, messages.NoRowsScored()) + } + } else { + fmt.Fprintln(w) + rows := make([][]string, 0, len(items)) + var rowsFailed, rowsUnscored int + for _, it := range items { + // One row per evaluated sample, not per verdict: a sample that + // failed three evaluators is one sample to go and look at, and + // listing it three times buries how much is actually wrong. + // + // An evaluator that returned no verdict is held apart from one that + // returned a failing verdict. Both keep the row, because the row is + // still worth looking at, but naming an errored evaluator among the + // ones the sample failed states something about the sample that + // nothing measured. + var failed, unjudged []string + reason := "" + for _, r := range it.Results { + switch { + case !r.Judged(): + unjudged = append(unjudged, r.Name) + case r.DidPass(): + continue + default: + failed = append(failed, r.Name) + } + if reason == "" { + reason = r.Reason + } + } + // The same predicate `-o json` filters on, so the two views of + // --failed-only cannot disagree about which rows went wrong. + if failedOnly && !it.Failed() { + continue + } + verdicts := strings.Join(failed, ", ") + if len(unjudged) > 0 { + note := strings.Join(unjudged, ", ") + " (no verdict)" + if verdicts == "" { + verdicts = note + } else { + verdicts += "; " + note + } + } + if verdicts == "" { + verdicts = "-" + } + if len(failed) > 0 { + rowsFailed++ + } else { + rowsUnscored++ + } + // No position column. It numbered within the current filter, so the + // same sample carried a different number depending on the flags while + // reading like an identifier -- and ITEM already carries the id, which + // is what `run output show` accepts. + rows = append(rows, []string{ + it.ID, + meanScoreOf(it.Results), + truncate(verdicts, 40), + truncate(reason, 44), + }) + } + // Only the first failure's reason fits a cell; `run output show` has + // the rest. + if err := emitTable(w, + []string{"ITEM", "SCORE", "EVALUATORS", "REASON"}, + rows); err != nil { + return err + } + // Counting unscored rows as failures put a number here that contradicted + // the totals two lines above, which is what a reader compares it with. + if failedOnly && len(rows) > 0 { + fmt.Fprint(w, messages.SamplesNeedingALook(rowsFailed, rowsUnscored)) + } + } + + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(w, messages.ReportLinkAfterRows(color.CyanString(url))) + } + return nil +} + +// truncate keeps a table readable when a reason runs to a paragraph. The full +// text is always in `-o json`. +func truncate(s string, n int) string { + s = strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") + if len(s) <= n { + return s + } + if n <= 1 { + return s[:n] + } + return s[:n-1] + "…" +} + +func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { + cw := csv.NewWriter(w) + + // Named as the service names it, and as the jsonl export already did, so a + // pipeline reading both formats needs one spelling rather than two. + if err := cw.Write([]string{"run_id", "status", "testing_criteria", "passed", "failed"}); err != nil { + return err + } + if len(run.PerTestingCriteria) == 0 { + if err := cw.Write([]string{run.ID, run.Status, "", "", ""}); err != nil { + return err + } + return flushCSV(cw) + } + for _, cr := range run.PerTestingCriteria { + if err := cw.Write([]string{ + run.ID, run.Status, cr.TestingCriteria, + strconv.Itoa(cr.Passed), strconv.Itoa(cr.Failed), + }); err != nil { + return err + } + } + return flushCSV(cw) +} + +// flushCSV reports what the buffer swallowed. +// +// csv.Writer buffers, so a disk that filled or a pipe that closed shows up only +// in Error() after the final Flush. Deferring the flush and returning nil made +// `run output export` report success over a file it had not finished writing. +func flushCSV(cw *csv.Writer) error { + cw.Flush() + return cw.Error() +} + +// Export formats. csv is the default because the results are a table and a +// build artifact is normally read by a spreadsheet or a diff. +const ( + formatCSV = "csv" + formatJSON = "json" + formatJSONL = "jsonl" +) + +// writeResultsJSONL emits one criterion per line, which is what a downstream +// job can stream without holding the whole run in memory. +func writeResultsJSONL(w io.Writer, run *eval_api.OpenAIEvalRun) error { + enc := json.NewEncoder(w) + if len(run.PerTestingCriteria) == 0 { + return enc.Encode(map[string]any{"run_id": run.ID, "status": run.Status}) + } + for _, cr := range run.PerTestingCriteria { + if err := enc.Encode(map[string]any{ + "run_id": run.ID, + "status": run.Status, + "testing_criteria": cr.TestingCriteria, + "passed": cr.Passed, + "failed": cr.Failed, + }); err != nil { + return err + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go new file mode 100644 index 00000000000..b3e0b8d5aa3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// twoCriteriaRun is a finished run with the shape export has to preserve: one +// row per testing criterion, all carrying the run they belong to. +func twoCriteriaRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 8, Failed: 2}, + {TestingCriteria: "coherence", Passed: 10, Failed: 0}, + }, + } +} + +// An export is read by a spreadsheet or a diff, so the header is part of the +// contract: renaming a column silently breaks whatever consumes it. +func TestWriteResultsCSV(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, twoCriteriaRun())) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + assert.Equal(t, []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_abc", "completed", "task_adherence", "8", "2"}, rows[1]) + assert.Equal(t, []string{"evalrun_abc", "completed", "coherence", "10", "0"}, rows[2]) + assert.Len(t, rows, 3, "one header and one row per criterion") +} + +// A run that graded nothing still has to produce a file with a header, because +// a consumer that gets zero bytes cannot tell an empty run from a failed +// export. +func TestWriteResultsCSV_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + require.Len(t, rows, 2) + assert.Equal(t, []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_empty", "failed", "", "", ""}, rows[1]) +} + +// A criterion name is service-supplied, so it can hold anything. The writer +// has to quote rather than corrupt the row. +func TestWriteResultsCSV_QuotesASeparatorInTheData(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: `groundedness, strict`, Passed: 1, Failed: 0}, + }, + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + require.Len(t, rows, 2) + assert.Equal(t, "groundedness, strict", rows[1][2], + "a comma in a criterion name must survive the round trip") +} + +// One criterion per line is what lets a downstream job stream results without +// holding the whole run. +func TestWriteResultsJSONL(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 2, "one line per criterion") + + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "evalrun_abc", first["run_id"]) + assert.Equal(t, "completed", first["status"]) + assert.Equal(t, "task_adherence", first["testing_criteria"]) + assert.EqualValues(t, 8, first["passed"]) + assert.EqualValues(t, 2, first["failed"]) + + var second map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[1]), &second)) + assert.Equal(t, "coherence", second["testing_criteria"]) +} + +// Every line has to parse on its own; that is the whole point of the format. +func TestWriteResultsJSONL_EachLineParsesAlone(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + for line := range strings.SplitSeq(strings.TrimSpace(buf.String()), "\n") { + var row map[string]any + assert.NoErrorf(t, json.Unmarshal([]byte(line), &row), "line is not self-contained: %s", line) + } +} + +func TestWriteResultsJSONL_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 1) + + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &row)) + assert.Equal(t, "evalrun_empty", row["run_id"]) + assert.Equal(t, "failed", row["status"]) + assert.NotContains(t, row, "testing_criteria", + "a run that graded nothing must not claim a criterion") +} + +// The three export formats are a documented set. A fourth spelling, or a +// missing one, is a promise broken on either side. +func TestExportFormatsAreTheDocumentedSet(t *testing.T) { + assert.Equal(t, "csv", formatCSV) + assert.Equal(t, "json", formatJSON) + assert.Equal(t, "jsonl", formatJSONL) + + usage := find(t, "run output export").Flags().Lookup("format") + require.NotNil(t, usage) + assert.Equal(t, formatCSV, usage.DefValue, + "results are a table, so the default artifact is the one a spreadsheet opens") + + for _, f := range []string{formatCSV, formatJSON, formatJSONL} { + assert.Containsf(t, usage.Usage, f, "--format accepts %q, so its help has to say so", f) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go new file mode 100644 index 00000000000..3b75a2fdf86 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// scoredRun is a run the way the service returns one, with rows attached. +func scoredRows() []eval_api.OutputItem { + return []eval_api.OutputItem{ + { + ID: "oi_1", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: new(true), Score: 5}, + {Name: "coherence", Passed: new(true), Score: 4}, + }, + }, + { + ID: "oi_2", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: new(false), Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: new(false), Score: 2, Reason: "Rambled."}, + }, + }, + } +} + +// One evaluated sample is one row. Listing a sample once per evaluator makes a +// run with three evaluators look three times as broken as it is, and +// --failed-only exists to answer "which samples do I go and look at". +func TestRenderResultsIsOneRowPerSample(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), false)) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "oi_2"), + "a sample that failed two evaluators must still be one row:\n%s", text) + + for _, header := range []string{"ITEM", "SCORE", "EVALUATORS", "REASON"} { + assert.Containsf(t, text, header, "the listing lost its %s column", header) + } + + // The old SAMPLE column numbered within the current filter, so the same + // sample carried a different number depending on the flags while reading + // like an identifier. ITEM carries the id `run output show` accepts. + assert.NotContains(t, text, "SAMPLE", + "a position that changes with the filter must not sit beside the id") + assert.NotContains(t, text, "FAILED EVALUATORS", + "the column also carries evaluators that returned no verdict, which did not fail") +} + +// The failing row has to name every evaluator that failed it, because that is +// what says whether the sample is broken or one evaluator is. +func TestRenderResultsNamesEveryFailedEvaluator(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), true)) + + text := out.String() + assert.Contains(t, text, "relevance, coherence") + assert.Contains(t, text, "Answered a different question.", + "the first failure's reason is what the row is looked at for") + assert.NotContains(t, text, "oi_1", "--failed-only must drop the passing sample") + assert.Contains(t, text, "1 sample(s) failed at least one evaluator.") +} + +// The footer is read against the totals printed a few lines above it, so it +// cannot count a row nothing scored as a row that failed. The reported run +// closed "13 sample(s) failed at least one evaluator" over totals that said 5 +// failed and 8 errored. +func TestFailedOnlyFooterHoldsUnscoredRowsApart(t *testing.T) { + items := []eval_api.OutputItem{ + {ID: "oi_fail", Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: new(false), Score: 1, Reason: "Answered a different question."}, + }}, + // No verdict: the evaluator errored on this row rather than scoring it. + {ID: "oi_unscored", Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, items, true)) + + text := out.String() + assert.Contains(t, text, "1 sample(s) failed at least one evaluator, and 1 could not be scored.", + "the two have to be counted apart:\n%s", text) + assert.NotContains(t, text, "2 sample(s) failed", + "an unscored row is not a failing one") + assert.Contains(t, text, "(no verdict)", + "and the row itself has to say which evaluator returned nothing") +} + +// The run summary carries pass and fail counts but no score, so the mean has +// to be averaged over the rows an evaluator actually scored. +func TestCriteriaMeans(t *testing.T) { + means := criteriaMeans(scoredRows()) + assert.InDelta(t, 3.0, means["relevance"], 0.001) + assert.InDelta(t, 3.0, means["coherence"], 0.001) + + assert.Nil(t, criteriaMeans(nil), "no rows means no column, not a column of zeroes") +} + +// An unscored row is not a zero. Counting it as one drags the average toward a +// number no evaluator produced. +func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { + rows := []eval_api.OutputItem{ + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: new(true)}}}, + {Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + // The zero value of a score is undefined, not 0.0. + rows[1].Results[0].Score = eval_api.LenientFloat(0) + + means := criteriaMeans(rows) + require.Contains(t, means, "relevance") + assert.InDelta(t, 2.0, means["relevance"], 0.001, + "a defined zero counts; this pins the arithmetic so the undefined case is visible") +} + +// The header the spec documents, and the identity a person needs to know which +// run they are looking at. +func TestRenderRunHeaderNamesTheEval(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", + EvalID: "eval_9", + Status: "completed", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + ResultCounts: &eval_api.EvalRunResultCounts{Total: 15, Passed: 12, Failed: 3}, + CreatedAt: float64(1785801525), + ModifiedAt: float64(1785802119), + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, map[string]float64{"relevance": 4.1})) + text := out.String() + + assert.Contains(t, text, "Run evalrun_9") + assert.Contains(t, text, "Eval support-agent-smoke", + "the declared name is what the author recognizes, not the service id") + assert.Contains(t, text, "Status completed") + assert.Contains(t, text, "Samples 15") + assert.Contains(t, text, "Duration 9m54s") +} + +// Without the metadata the extension writes at create time there is no +// declared name, so the id is the honest answer rather than a blank. +func TestRenderRunHeaderFallsBackToTheEvalID(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_9", EvalID: "eval_9", Status: "queued"} + require.NoError(t, renderRun(&out, run, nil)) + assert.Contains(t, out.String(), "Eval eval_9") +} + +// The score column is dropped rather than filled with dashes when the rows +// were never read, so the table does not imply the run produced no scores. +func TestRenderRunOmitsTheScoreColumnWithoutMeans(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{{TestingCriteria: "relevance", Passed: 2}}, + } + + var without bytes.Buffer + require.NoError(t, renderRun(&without, run, nil)) + assert.NotContains(t, without.String(), "MEAN SCORE") + + var with bytes.Buffer + require.NoError(t, renderRun(&with, run, map[string]float64{"relevance": 4.15})) + assert.Contains(t, with.String(), "MEAN SCORE") + assert.Contains(t, with.String(), "4.2", "the mean is shown to one decimal") +} + +// `run output show` is what a person opens after a failing listing, so it has +// to be readable. It used to emit raw JSON whatever was asked for. +func TestRenderOutputItemIsNotJSON(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_01JQZY7K3R", + RunID: "evalrun_1", + Status: "fail", + Results: []eval_api.OutputResult{{ + Name: "builtin.task_adherence", + Score: 0.35, + Passed: new(false), + Reason: "Task abandoned after the first clarifying question.", + }}, + })) + + text := out.String() + assert.NotContains(t, text, `"results"`, "the detail view must not be JSON") + for _, want := range []string{ + "Item", "oi_01JQZY7K3R", + "Status", "fail", + "builtin.task_adherence", "0.35", + "Task abandoned after the first clarifying question.", + } { + assert.Contains(t, text, want) + } +} + +// The listing truncates the reason to a cell, so this view exists to carry the +// whole of it. Cutting it here would leave it readable nowhere. +func TestRenderOutputItemKeepsTheWholeReason(t *testing.T) { + reason := strings.Repeat("a reason that runs well past any column width. ", 8) + + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "fail", + Results: []eval_api.OutputResult{{Name: "relevance", Passed: new(false), Reason: reason}}, + })) + + assert.Contains(t, out.String(), reason) +} + +// A rubric reports one result per dimension, all carrying the evaluator's +// name. Printed flat they read as several evaluators that share a name. +func TestRenderOutputItemGroupsARubricsDimensions(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "fail", + Results: []eval_api.OutputResult{ + {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: new(false)}, + {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: new(true)}, + {Name: "builtin.task_adherence", Score: 0.35, Passed: new(false)}, + }, + })) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "support-agent-quality"), + "the evaluator is named once, above its dimensions:\n%s", text) + for _, want := range []string{"resolves_issue", "cites_policy", "builtin.task_adherence"} { + assert.Contains(t, text, want) + } +} + +// The service echoes the evaluator's name in `metric` for a single-score +// evaluator. Nesting that reads as a dimension that happens to share its +// evaluator's name. +func TestRenderOutputItemDoesNotNestASelfNamedMetric(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "completed", + Results: []eval_api.OutputResult{ + {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: new(true)}, + }, + })) + + assert.Equal(t, 1, strings.Count(out.String(), "task_adherence"), + "the evaluator must be named once:\n%s", out.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go new file mode 100644 index 00000000000..646fd560e50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exit code is the whole contract with a pipeline, and there are three +// answers it has to be able to give: the evaluation ran and passed, it ran and +// regressed, or it could not run. The gate owns the middle one; this owns the +// last. +// +// Reporting a run that errored and then exiting 0 tells the pipeline the +// evaluation passed, which is the one answer that is never true. +func TestRunCompleted(t *testing.T) { + require.NoError(t, runCompleted(nil), + "nothing was waited for, so there is nothing to report") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "completed"})) + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "Completed"}), + "the service is not consistent about case") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1"}), + "a status the service did not send is not a failure to report") + + for _, status := range []string{"failed", "error", "canceled", "cancelled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "run_abc", Status: status}) + require.Error(t, err, "status %q must not exit 0", status) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), status, + "the message has to name the status, which is what the caller acts on") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go new file mode 100644 index 00000000000..a26d7e9b4a5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// finishedRun is a run the way the service returns one: counts over samples, +// and a result per testing criterion. +func finishedRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc123", + Status: "completed", + ResultCounts: &eval_api.EvalRunResultCounts{ + Total: 10, Passed: 7, Failed: 3, + }, + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 9, Failed: 1}, + {TestingCriteria: "coherence", Passed: 7, Failed: 3}, + }, + } +} + +// The whole point of waiting for a run is the verdict per evaluator. Printing +// only the status meant the answer to the question the command was asked took +// a second command to see. +func TestRenderRunReportsEveryEvaluator(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + text := out.String() + + assert.Contains(t, text, "evalrun_abc123") + assert.Contains(t, text, "completed") + + for _, criterion := range []string{"relevance", "coherence"} { + assert.Contains(t, text, criterion, + "every evaluator the run scored must appear") + } + assert.Contains(t, text, "90.0%", "relevance passed 9 of 10") + assert.Contains(t, text, "70.0%", "coherence passed 7 of 10") + assert.Contains(t, text, "7/10", "the sample counts must be shown, not just the rate") +} + +// Two runs of the same eval have to read the same way. The service returns the +// criteria in whatever order it evaluated them, which is not stable. +func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + + text := out.String() + assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), + "evaluators must be listed in a stable order") +} + +// An errored row is not a failing row: the evaluator never reached a verdict. +// Folding the two together would report a service problem as a quality problem. +func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { + run := finishedRun() + run.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 1, Errored: 2} + run.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 7, Failed: 1, Errored: 2}, + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, nil)) + text := out.String() + + assert.Contains(t, text, "2 errored") + assert.Contains(t, text, "87.5%", + "the pass rate is over what was scored, not over what was attempted") + assert.Contains(t, text, "errored and were not scored") +} + +// A rate over nothing is not zero. Printing 0.0% for a criterion that scored +// no rows reads as a total failure rather than as no data. +func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { + assert.Equal(t, "-", formatRate(0, 0)) + assert.Equal(t, "0.0%", formatRate(0, 4)) + assert.Equal(t, "100.0%", formatRate(4, 4)) + assert.Equal(t, "33.3%", formatRate(1, 3)) +} + +// The next thing anyone does after seeing failures is look at them, so the +// command that shows them is named — and it has to be a command that exists. +func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") + + clean := finishedRun() + clean.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 10} + clean.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 10}, + } + var cleanOut bytes.Buffer + require.NoError(t, renderRun(&cleanOut, clean, nil)) + assert.NotContains(t, cleanOut.String(), "--failed-only", + "a run with nothing to look at must not send anyone looking") +} + +// A run that never produced counts still has to render. The service returns +// none for a run that failed before scoring, and a nil dereference there would +// replace the failure message with a panic. +func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"}, nil)) + assert.Contains(t, out.String(), "evalrun_x") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go new file mode 100644 index 00000000000..55329762df9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorSchemasIncludesBuiltins covers the function that supplies +// the schemas, rather than the builder that consumes them. +// +// The builder was already tested against every built-in, but the test fetched +// them itself with the Builtin filter. Production did not: it listed +// unfiltered, which returns only the project's own evaluators, so every +// built-in reached the builder with no schema at all. The builder was correct +// and the criteria were still wrong, and no test could see it because each one +// constructed the input production was failing to construct. +func TestLiveEvaluatorSchemasIncludesBuiltins(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + // The listing production used to rely on, to show what it omits. + unfiltered, err := client.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + require.NoError(t, err) + builtinsInUnfiltered := 0 + for _, e := range unfiltered.Value { + if eval_api.IsBuiltinEvaluator(e.Name) { + builtinsInUnfiltered++ + } + } + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "no evaluator schemas were resolved at all") + + builtins := 0 + for name, summary := range schemas { + if !eval_api.IsBuiltinEvaluator(name) { + continue + } + builtins++ + assert.NotNil(t, summary.DataSchema(), + "%s resolved without the contract the criteria are shaped from", name) + } + + require.NotZero(t, builtins, + "built-ins must be resolvable; the unfiltered listing returns %d of them, "+ + "so they have to be asked for by type", builtinsInUnfiltered) +} + +// The fields an evaluator declares are the ones its criterion has to bind, so +// a conversation-level evaluator must resolve to its conversation field. +func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + var name string + for n, summary := range schemas { + if eval_api.IsBuiltinEvaluator(n) && summary.SupportsLevel("conversation") { + if ds := summary.DataSchema(); ds != nil && ds.Accepts(conversationField) { + name = n + break + } + } + } + if name == "" { + t.Skip("no built-in advertises a conversation contract on this project") + } + + plan, err := planCriterion( + evalcore.EvaluatorRef{ + Name: name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }, + schemas[name], + nil, // no target: the dataset holds both sides of the exchange + map[string]bool{conversationField: true}, + "conversation", + ) + require.NoError(t, err) + assert.Equal(t, "{{item."+conversationField+"}}", plan.dataMapping[conversationField], + "%s must bind its conversation field", name) + assert.NotEmpty(t, plan.dataMapping, "an empty mapping scores nothing") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go new file mode 100644 index 00000000000..1c65d16c8b5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "math" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An evaluator that errored on a row still sends a result, and its score +// decodes to NaN. criteriaMeans has always skipped those -- its comment says +// counting them as zero "would drag the average toward a number no evaluator +// produced" -- but the per-sample column averaged them in, so one errored +// evaluator made the whole sample read NaN. +func TestASampleScoreLeavesOutWhatWasNotScored(t *testing.T) { + undefined := eval_api.LenientFloat(math.NaN()) + + cases := []struct { + name string + results []eval_api.OutputResult + want string + }{ + { + name: "no results at all", + results: nil, + want: "-", + }, + { + name: "every evaluator scored", + results: []eval_api.OutputResult{ + {Score: 1.0}, {Score: 0.5}, + }, + want: "0.75", + }, + { + name: "one evaluator errored", + results: []eval_api.OutputResult{ + {Score: 4.0}, {Score: undefined}, + }, + want: "4.00", + }, + { + name: "nothing was scored", + results: []eval_api.OutputResult{ + {Score: undefined}, {Score: undefined}, + }, + want: "-", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, meanScoreOf(tc.results)) + }) + } +} + +// The same rule for a single score: a dash says "not scored", where NaN says +// nothing a reader can use. +func TestAScoreThatIsNotANumberShowsAsAbsent(t *testing.T) { + assert.Equal(t, "-", formatScore(eval_api.LenientFloat(math.NaN()))) + assert.Equal(t, "-", formatScore(eval_api.LenientFloat(math.Inf(1)))) + assert.Equal(t, "0.75", formatScore(eval_api.LenientFloat(0.75))) + assert.Equal(t, "0.00", formatScore(eval_api.LenientFloat(0)), + "a real zero is a score and must still be shown") +} + +// The gate compares exact values while the message rounds, so a rate just under +// the threshold could be reported as below itself. The spec's hero scenario +// shows one decimal, so that is kept for every case where the two differ. +func TestAGateBreachNeverReportsARateAsBelowItself(t *testing.T) { + gate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + breach := gate.breach(&eval_api.EvalRunResultCounts{Total: 10000, Passed: 7996, Failed: 2004}) + require.NotEmpty(t, breach, "7996/10000 is under 0.8 and must breach") + assert.NotContains(t, breach, "80.0% is below the required 80.0%", + "a line saying a rate is below itself tells a reader nothing") + assert.Contains(t, breach, "79.96%") + + // The wording the spec shows is unchanged wherever rounding does not collide. + assert.Equal(t, + "pass rate 76.4% is below the required 80.0%", + gate.breach(&eval_api.EvalRunResultCounts{Total: 1000, Passed: 764, Failed: 236})) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go new file mode 100644 index 00000000000..9a5f26b3372 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -0,0 +1,617 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The command surface is a contract with the spec and with the sibling Foundry +// extensions, and it is the part of this tool users type from memory. Nothing +// was checking it: the flag that writes results to a file was `--out-file` +// while the spec, Scenario 4, and `azd ai skill download` all say +// `--output-file`, and it took reading the two documents side by side to see. +// +// These tests walk the built tree, so a command or flag that is renamed, +// dropped, or quietly added has to be acknowledged here. + +// walk visits every command in the tree, skipping the ones azd contributes. +func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { + t.Helper() + for _, child := range cmd.Commands() { + name := strings.Fields(child.Use)[0] + switch name { + case "help", "completion", "listen", "metadata": + continue + } + full := append(append([]string{}, path...), name) + visit(strings.Join(full, " "), child) + walk(t, child, full, visit) + } +} + +// commandTree is every command the extension exposes, and is the surface the +// spec's command table describes. +func TestCommandTreeMatchesTheSpec(t *testing.T) { + want := []string{ + "dataset", + "dataset create", + "dataset delete", + "dataset list", + "dataset show", + "dataset update", + "dataset versions", + "dataset versions list", + "create", + "delete", + "evaluator", + "evaluator create", + "evaluator delete", + "evaluator list", + "evaluator show", + "evaluator update", + "evaluator versions", + "evaluator versions list", + // One generate for both artifacts, and one job group for both + // collections, selected by --dataset / --evaluator. + "generate", + "job", + "job cancel", + "job delete", + "job list", + "job show", + "init", + "list", + "run", + "run cancel", + "run delete", + "run list", + "run output", + "run output export", + "run output list", + "run output show", + "run show", + "run start", + "show", + } + + var got []string + walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { + got = append(got, path) + }) + + assert.ElementsMatch(t, want, got, + "the command tree changed; update the spec's command table with it") +} + +// Flag names are shared vocabulary across the Foundry extensions. A command +// that invents its own spelling for something the others already name is the +// kind of difference nobody notices until a user types the one they learned +// somewhere else. +func TestFlagVocabularyIsShared(t *testing.T) { + // Meaning → the one spelling for it, from the spec's vocabulary table. + // A command that means one of these must use exactly this name, and the + // near-misses are listed so a rename back is caught rather than accepted. + forbidden := map[string]string{ + "--out-file": "--output-file", + "--out-dir": "--output-dir", + "--file": "--from-file", + "--rubric": "--from-file", + "--from-traces": "deferred to M2", + "--response-id": "deferred to M2", + "--no-target": "deferred to M2", + "--out": "--output-file", + "--dir": "--output-dir", + "--baseline": "deferred to M2", + "--cron": "deferred to M2", + "--folder": "deferred to M2", + "--init-params": "deferred to M2", + "--data-schema": "deferred to M2", + "--metrics": "deferred to M2", + "--trace-window": "deferred to M2", + "--max-turns": "deferred to M2", + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if want, bad := forbidden["--"+f.Name]; bad { + t.Errorf("%s declares --%s; use %s", path, f.Name, want) + } + }) + }) +} + +// M1 promises `-o json` and `--no-prompt` throughout. Both come from the azd +// extension SDK's root command, so every command inherits them — until one +// declares its own flag by the same name, which silently shadows the global +// and leaves that one command unable to answer in JSON or to run unattended. +func TestNoCommandShadowsAGlobalFlag(t *testing.T) { + global := []string{"output", "no-prompt", "environment", "cwd", "debug"} + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + for _, name := range global { + assert.Nilf(t, cmd.LocalFlags().Lookup(name), + "%s declares its own --%s, which shadows the global one", path, name) + } + }) +} + +// The two commands that write a file have to agree on what that flag is +// called, and it has to be the name the sibling extensions use. +func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { + for _, path := range []string{"run output list", "run output export"} { + cmd := find(t, path) + require.NotNil(t, cmd.Flags().Lookup("output-file"), + "%s must write to --output-file, the name `azd ai skill download` uses", path) + assert.Nil(t, cmd.Flags().Lookup("out-file"), + "%s must not keep the old spelling alongside the shared one", path) + } +} + +// `init` is the one command with a documented flag table, so it is pinned +// whole: an extra flag there is a promise the spec does not make, and a +// missing one is a promise it does. +func TestInitFlagsMatchTheSpec(t *testing.T) { + cmd := find(t, "init") + + var got []string + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name != "help" { + got = append(got, "--"+f.Name) + } + }) + + assert.ElementsMatch(t, []string{ + "--name", "--target", "--source", "--dataset", "--max-traces", + "--evaluator", "--judge-model", "--path", "--force", + }, got, "init's flags are a table in the spec; change both together") +} + +// `init` makes no service calls, so it must not offer the flag that says where +// to make them. +func TestInitTakesNoProjectEndpoint(t *testing.T) { + assert.Nil(t, find(t, "init").Flags().Lookup("project-endpoint"), + "init is offline; a project endpoint would imply otherwise") +} + +// Every command that does reach the service accepts it, because the shared +// Foundry resolver is how a project is named without an azd environment. +// --eval takes "the name of the eval declared in the configuration", which +// means the command has to be able to find that configuration. The only other +// way it can is a path `init` recorded in the azd environment, which a +// --project-endpoint caller does not have -- so a configuration outside ./evals +// could be started and then not listed, shown or cancelled. +func TestEveryCommandTakingAnEvalNameCanBeToldWhereTheConfigIs(t *testing.T) { + var checked int + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.Flags().Lookup("eval") == nil { + return + } + checked++ + assert.NotNilf(t, cmd.Flags().Lookup("path"), + "%s resolves a declared eval name, so it must accept --path", path) + }) + assert.NotZero(t, checked, "no command took --eval, so this checked nothing") +} + +func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { + groups := map[string]bool{ + "dataset": true, "evaluator": true, "run": true, + "job": true, "run output": true, + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.RunE == nil || path == "init" { + return + } + if groups[path] { + return + } + assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), + "%s reaches the service, so it must accept --project-endpoint", path) + }) +} + +// The spec says --from "selects one or more of the four sources", so it has to +// be repeatable. Declared as a plain string it would still accept every +// documented single-source invocation and silently keep only the last of a +// repeated one, which is the kind of difference no example in the spec shows. +func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "generate").Flags().Lookup("from") + require.NotNil(t, flag, "generate must offer --from") + + assert.Equal(t, "stringSlice", flag.Value.Type(), + "--from selects one or more sources, so it cannot be a single string") +} + +// `--from` names sources; the set it accepts is the set the service has a path +// for, and the help has to list exactly that set. +func TestGenerateFromListsEverySource(t *testing.T) { + usage := find(t, "generate").Flags().Lookup("from").Usage + + for _, source := range project.GenerateSources { + assert.Containsf(t, usage, source, + "--from accepts %q, so its help has to say so", source) + } +} + +// One command now generates both artifacts, but --from and --max-samples shape +// the dataset only. The help has to say so, or they read as applying to the +// rubric as well. +func TestGenerateSaysWhichFlagsAreDatasetOnly(t *testing.T) { + flags := find(t, "generate").Flags() + + for _, name := range []string{"from", "max-samples"} { + flag := flags.Lookup(name) + require.NotNilf(t, flag, "generate must offer --%s", name) + assert.Containsf(t, flag.Usage, "Dataset only", + "--%s shapes the dataset only, so its help has to say so", name) + } +} + +// The selector narrows generation; omitting both is the zero-to-first-eval +// path, so neither flag may be required. +func TestGenerateSelectorIsOptional(t *testing.T) { + cmd := find(t, "generate") + + for _, name := range []string{"dataset", "evaluator"} { + flag := cmd.Flags().Lookup(name) + require.NotNilf(t, flag, "generate must offer --%s", name) + assert.Equal(t, "false", flag.DefValue, + "--%s is off by default, which is what generates both", name) + } +} + +// The spec's run table says which commands carry which flag. Where it says +// "every", that is checkable; where it names two commands, a third carrying the +// flag is a promise the spec does not make and a missing one is a promise it +// does. +// +// This pins placement, not the whole flag list: unlike init's, the run table is +// headed "Flag | Commands | Default" and documents defaults rather than +// enumerating every flag. +func TestRunFlagsSitWhereTheSpecSaysTheyDo(t *testing.T) { + // Flag → exactly the run commands that may declare it. nil means every + // run command that does something. + placement := map[string][]string{ + "eval": nil, + "dataset": {"run start"}, + "fail-on": {"run start", "run show"}, + "wait": {"run start", "run show"}, + "format": {"run output export"}, + } + + // Every run command that actually runs, which is what "every run command" + // means — the bare groups take no flags. + var runCommands []string + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if strings.HasPrefix(path, "run") && cmd.RunE != nil { + runCommands = append(runCommands, path) + } + }) + require.NotEmpty(t, runCommands) + + for flag, allowed := range placement { + if allowed == nil { + allowed = runCommands + } + for _, path := range runCommands { + has := find(t, path).Flags().Lookup(flag) != nil + want := slices.Contains(allowed, path) + + switch { + case want && !has: + t.Errorf("%s must accept --%s; the spec's run table says so", path, flag) + case !want && has: + t.Errorf("%s declares --%s, which the spec gives only to %s", + path, flag, strings.Join(allowed, ", ")) + } + } + } +} + +// `--eval` is how a run command finds the eval, and the spec gives it to every +// one of them. Losing it from a single command makes that command unusable in a +// project with more than one eval. +func TestEveryRunCommandTakesEval(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if !strings.HasPrefix(path, "run") || cmd.RunE == nil { + return + } + assert.NotNilf(t, cmd.Flags().Lookup("eval"), + "%s must accept --eval, which the spec gives to every run command", path) + }) +} + +// The tagged suites drive the binary by writing flags as strings, so a flag +// that is renamed or removed still compiles there and only fails when someone +// has the credentials to run them. +// +// That is not hypothetical. Removing `--eval-id` and the generation spec file +// left 28 uses of `--eval-id` and two `--config` tests behind in tests/cli, +// every one of which would have failed at the first live run — under `live` and +// `hero` tags that `go test ./...` never builds. This checks them from the +// default suite, where a rename is caught by the person doing the renaming. +func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { + // Every flag any command declares, plus the globals azd contributes. + known := map[string]bool{ + "output": true, "no-prompt": true, "environment": true, + "cwd": true, "debug": true, "help": true, + } + walk(t, NewRootCommand(), nil, func(_ string, cmd *cobra.Command) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + + // Only string literals, which is how a test spells a flag it passes to the + // binary. Prose in a comment is not a flag. + literal := regexp.MustCompile(`"--([a-z][a-z0-9-]*)"`) + + err := filepath.WalkDir("../../tests", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range literal.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "%s:%d passes --%s, which no command declares", path, i+1, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + +// A suggestion that names a flag has to name one the suggested command takes. +// +// `run start --no-wait` closed with "Reattach with: azd ai eval run show +// --eval-id " for the whole life of the branch that removed --eval-id. +// The command resolved, so the suggestion check passed; the flag did not exist, +// so the one line a user is told to paste was the one guaranteed to fail. +func TestSuggestedFlagsExist(t *testing.T) { + // `azd ai eval ... --flag`, with the flag anywhere after it. + suggestion := regexp.MustCompile(`azd ai eval ((?:[a-z][a-z0-9-]*\s+)+)([^"'\n]*)`) + flagName := regexp.MustCompile(`--([a-z][a-z0-9-]*)`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range suggestion.FindAllStringSubmatch(line, -1) { + flags := flagName.FindAllStringSubmatch(m[2], -1) + if len(flags) == 0 { + continue + } + // Longest command prefix that resolves; the rest is arguments. + words := strings.Fields(m[1]) + for len(words) > 0 { + if resolved, _, e := NewRootCommand().Find(words); e == nil && + strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + words = words[:len(words)-1] + } + if len(words) == 0 { + continue // the command itself is checked above + } + cmd, _, _ := NewRootCommand().Find(words) + for _, f := range flags { + assert.NotNilf(t, cmd.Flags().Lookup(f[1]), + "%s:%d suggests `azd ai eval %s --%s`, which that command does not accept", + path, i+1, strings.Join(words, " "), f[1]) + } + } + } + return nil + }) + require.NoError(t, err) +} + +// siblingNamespaces are the other Foundry extensions this one points users at. +// Listed rather than wildcarded so a typo in a namespace still fails. +var siblingNamespaces = map[string]bool{ + "project": true, // `azd ai project set` owns the shared endpoint context + "dataset": true, // where the dataset commands go once they move + "agent": true, +} + +// find resolves a command path, failing the test when it does not exist. +func find(t *testing.T, path string) *cobra.Command { + t.Helper() + cmd, _, err := NewRootCommand().Find(strings.Fields(path)) + require.NoError(t, err, "no such command: %s", path) + require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], + strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) + return cmd +} + +// Messages that tell a user what to run next have to name a command that +// exists. +// +// Rebuilding the surface left `run start --no-wait` closing with "Check +// progress with: azd ai eval results show", a command that had been renamed +// out of existence — so the one instruction printed at the moment a user needs +// it was the one thing guaranteed to fail. Nothing catches that: the string +// compiles, the command that prints it succeeds, and only someone following +// the advice finds out. +// This extension's namespace is `ai.eval`, so every command it can suggest +// begins `azd ai eval`. Anchoring on that prefix is what caught the renamed +// command above — and anchoring only on it is what let three suggestions +// through pointing at `azd ai dataset`, a namespace no installed extension +// serves. So the prefix checked is `azd ai`, and anything under it that is not +// this extension's own is a command nobody can run. +func TestSuggestedCommandsExist(t *testing.T) { + root := "../.." + pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source + if err != nil { + return err + } + + for line := range strings.SplitSeq(string(body), "\n") { + // Comments explain the surface; only what reaches a terminal has + // to resolve. + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllStringSubmatch(line, -1) { + words := strings.Fields(m[1]) + if len(words) == 0 { + continue + } + + // A suggestion under a sibling's namespace is that extension's + // contract and cannot be resolved from here. Only the ones this + // extension actually points at are allowed, so a typo still + // fails rather than passing as "probably somebody else's". + if siblingNamespaces[words[0]] { + continue + } + + // `ai.eval` is this extension's namespace, so it is the only + // other thing under `azd ai` that can resolve. + if words[0] != "eval" { + t.Errorf("%s suggests `azd ai %s`, which is neither this "+ + "extension's namespace nor a sibling it knows about", path, m[1]) + continue + } + words = words[1:] + + // Trim trailing prose: "run start" is a command, "run start + // and summarize" is a sentence that begins with one. + for len(words) > 0 { + if _, _, err := NewRootCommand().Find(words); err == nil { + resolved, _, _ := NewRootCommand().Find(words) + if strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + } + words = words[:len(words)-1] + } + assert.NotEmpty(t, words, + "%s suggests `azd ai %s`, which is not a command", path, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + +// Every flag a message names has to exist on some command. +// +// TestSuggestedFlagsExist only looks inside a quoted `azd ai eval ...` command, +// so a message that names a flag on its own escapes it. One did: +// DatasetHasUnregisteredEdits told the reader to pass `--eval-id `, a flag +// removed long before, and the check above saw no command to attach it to. +func TestBareFlagsInMessagesExist(t *testing.T) { + flagRef := regexp.MustCompile(`--([a-z][a-z0-9-]{1,})`) + + known := map[string]bool{} + root := NewRootCommand() + root.PersistentFlags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + walk(t, root, nil, func(_ string, c *cobra.Command) { + c.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + // azd's own, which messages legitimately name. + for _, global := range []string{"cwd", "debug", "environment", "no-prompt", "output", "help"} { + known[global] = true + } + + body, err := os.ReadFile(filepath.Join("..", "messages", "messages.go")) + require.NoError(t, err) + + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range flagRef.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "messages.go:%d names --%s, which no command accepts", i+1, m[1]) + } + } +} + +// A command suggested with an argument has to be suggested with the argument +// filled in. +// +// `--no-wait` exists so the caller can walk away, and the line they walk away +// with is the one they paste when they come back. Printing +// `azd ai eval job show ` reads like a command and is not one: it +// resolves, so the check above passes, and it fails the moment anyone uses it. +func TestSuggestedCommandsCarryNoPlaceholders(t *testing.T) { + placeholder := regexp.MustCompile(`azd ai eval [^"'\n]*<[a-z-]+>`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + // A `Use:` string and the help text around it are where a + // placeholder belongs: cobra prints it as the signature. + if strings.HasPrefix(trimmed, "//") || + strings.HasPrefix(trimmed, "Use:") || + strings.HasPrefix(trimmed, "Short:") || + strings.HasPrefix(trimmed, "Long:") { + continue + } + if m := placeholder.FindString(line); m != "" { + t.Errorf("%s:%d suggests %q; substitute the value instead", + path, i+1, m) + } + } + return nil + }) + require.NoError(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go new file mode 100644 index 00000000000..92b18ee230a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A list view is uppercase headers over a rule, per the spec's output +// conventions and the sibling extension it cites. The rule is what separates +// the header from the data at a glance, and every `list` command was printing +// the header straight onto the first row. +func TestEmitTableWritesTheRule(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "VERSION"}, + [][]string{{"support-regression", "3"}, {"nightly", "1"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4, "a header, its rule, and one line per row") + + assert.Contains(t, lines[0], "NAME") + assert.Contains(t, lines[0], "VERSION") + + // Dashes as wide as the header they sit under, which is what makes the + // rule line up once tabwriter has padded the columns. + assert.Contains(t, lines[1], strings.Repeat("-", len("NAME"))) + assert.Contains(t, lines[1], strings.Repeat("-", len("VERSION"))) + assert.Empty(t, strings.Trim(lines[1], "- "), + "the rule carries nothing but dashes and padding") + + assert.Contains(t, lines[2], "support-regression") + assert.Contains(t, lines[3], "nightly") +} + +// The columns line up: the rule is padded to the same widths as the header, so +// a wide value in the first row does not leave the rule short. +func TestEmitTableRuleAlignsWithTheHeader(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "STATUS"}, + [][]string{{"a-very-much-longer-value-than-the-header", "completed"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3) + + // tabwriter pads every line in a column to the same width, so the header + // and its rule start their second column at the same offset. + assert.Equal(t, + strings.Index(lines[0], "STATUS"), + strings.Index(lines[1], "------"), + "the rule has to sit under the header it belongs to") +} + +// A listing with nothing in it still prints the header and rule: a caller +// seeing no output cannot tell an empty list from a command that failed to +// render. +func TestEmitTableWithNoRows(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, []string{"NAME", "VERSION"}, nil)) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + assert.Len(t, lines, 2) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go new file mode 100644 index 00000000000..3e2a177d546 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package exterrors + +// The codes azd renders alongside an error's category and suggestion. +// +// Only the ones this extension actually raises are listed. This file used to +// carry the toolbox and skill vocabulary it was copied from -- 37 codes for +// resources this extension has no concept of -- which offered anyone looking +// for the right code a menu belonging to a different product. + +// Error codes for user cancellation. +const ( + CodeCancelled = "cancelled" +) + +// Error codes for validation failures (user input, manifests, flags). +const ( + CodeInvalidParameter = "invalid_parameter" +) + +// Error codes for dependency failures (missing resources, services, env values). +const ( + CodeMissingProjectEndpoint = "missing_project_endpoint" +) + +// Error codes for auth failures. +const ( + CodeLoginExpired = "login_expired" + CodeAuthFailed = "auth_failed" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go new file mode 100644 index 00000000000..d6428a6dfa1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package exterrors provides structured error helpers for the +// azure.ai.evaluations extension. +// +// Use plain Go errors until the current code can confidently choose a final +// category, code, and suggestion. At that point, create a structured error with +// one of the helpers in this package or with [ServiceFromAzure] for Azure SDK +// failures. +// +// Once an error is structured, usually return it unchanged. Avoid wrapping a +// structured error with [fmt.Errorf] and %w for extra context: azd serializes +// the structured error's own message and metadata, not the outer wrapper text. +package exterrors + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --------------------------------------------------------------------------- +// Structured error factories +// --------------------------------------------------------------------------- + +// Validation returns a validation [azdext.LocalError] for user input / flag errors. +func Validation(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryValidation, + Suggestion: suggestion, + } +} + +// Dependency returns a dependency [azdext.LocalError] for missing resources or services. +func Dependency(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryDependency, + Suggestion: suggestion, + } +} + +// Auth returns an auth [azdext.LocalError] for authentication/authorization failures. +func Auth(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryAuth, + Suggestion: suggestion, + } +} + +// User returns a user-action [azdext.LocalError] (e.g. cancellation). No suggestion. +func User(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryUser, + } +} + +// Internal returns an internal [azdext.LocalError] for unexpected extension failures. +func Internal(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryInternal, + } +} + +// Cancelled returns a user cancellation error. +func Cancelled(message string) error { + return User(CodeCancelled, message) +} + +// --------------------------------------------------------------------------- +// Azure error converters +// --------------------------------------------------------------------------- + +// ServiceFromAzure wraps an [azcore.ResponseError] into an [azdext.ServiceError] +// with operation context. If the error is not an azcore.ResponseError, it +// returns a generic internal [azdext.LocalError]. +func ServiceFromAzure(err error, operation string) error { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + serviceName := "" + if respErr.RawResponse != nil && respErr.RawResponse.Request != nil { + serviceName = respErr.RawResponse.Request.Host + } + code := respErr.ErrorCode + if code == "" { + code = fmt.Sprintf("%d", respErr.StatusCode) + } + return &azdext.ServiceError{ + Message: fmt.Sprintf("%s: %s", operation, respErr.Error()), + ErrorCode: fmt.Sprintf("%s.%s", operation, code), + StatusCode: respErr.StatusCode, + ServiceName: serviceName, + } + } + if IsCancellation(err) { + return Cancelled(fmt.Sprintf("%s was cancelled", operation)) + } + return Internal(operation, fmt.Sprintf("%s: %s", operation, err.Error())) +} + +// FromPrompt wraps a gRPC error from an azd host Prompt call into a structured +// error. Auth errors (Unauthenticated) are classified as Auth errors with a +// re-auth suggestion; cancellations as User cancellations; other errors are +// returned wrapped with the provided context message. +func FromPrompt(err error, contextMsg string) error { + if err == nil { + return nil + } + + if IsCancellation(err) { + return Cancelled(contextMsg) + } + + st, ok := status.FromError(err) + if ok && st.Code() == codes.Unauthenticated { + return Auth( + CodeAuthFailed, + fmt.Sprintf("%s: %s", contextMsg, st.Message()), + "run `azd auth login` to authenticate", + ) + } + + return fmt.Errorf("%s: %w", contextMsg, err) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// IsCancellation reports whether err represents user cancellation +// ([context.Canceled] or gRPC [codes.Canceled]). +func IsCancellation(err error) bool { + if errors.Is(err, context.Canceled) { + return true + } + if st, ok := status.FromError(err); ok && st.Code() == codes.Canceled { + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go new file mode 100644 index 00000000000..1be5ee0deb2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeEnv answers the two reads with whatever the case under test needs. +type fakeEnv struct { + current *azdext.EnvironmentResponse + currentErr error + values map[string]string + valueErr map[string]error + asked []string +} + +func (f *fakeEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + return f.current, f.currentErr +} + +func (f *fakeEnv) GetValue( + _ context.Context, req *azdext.GetEnvRequest, _ ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + f.asked = append(f.asked, req.Key) + if err, ok := f.valueErr[req.Key]; ok { + return nil, err + } + return &azdext.KeyValueResponse{Key: req.Key, Value: f.values[req.Key]}, nil +} + +func envNamed(name string) *azdext.EnvironmentResponse { + return &azdext.EnvironmentResponse{Environment: &azdext.Environment{Name: name}} +} + +// An answer of "nothing here" leaves the cascade free to carry on; a failure to +// answer has to stop it, because carrying on resolves to a lower-priority +// endpoint that can belong to a different project. +// +// This is the rule that has regressed twice while every test passed, because +// the only seam was the whole function. +func TestReadEnvHostedSource_TellsAbsenceApartFromFailure(t *testing.T) { + cases := []struct { + name string + env *fakeEnv + wantValue string + wantName string + wantErr string + }{ + { + name: "no environment selected", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "default environment not found")}, + }, + { + name: "outside a project altogether", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`")}, + }, + { + name: "the environment named in config is gone", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, "'dev': environment not found")}, + }, + { + name: "no daemon", + env: &fakeEnv{currentErr: status.Error(codes.Unavailable, "connection refused")}, + }, + { + name: "the login has expired", + env: &fakeEnv{currentErr: status.Error(codes.Unauthenticated, "expired")}, + wantErr: "expired", + }, + { + name: "the daemon broke while looking", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "loading project state: permission denied")}, + wantErr: "loading project state", + }, + { + name: "the foundry key answers", + env: &fakeEnv{current: envNamed("dev"), values: map[string]string{foundryEnvKey: "https://a"}}, + wantValue: "https://a", + wantName: "dev", + }, + { + // The key `azd ai agent init` and `azd add` persist, read only + // when the newer one has nothing. + name: "the older key answers when the newer one is empty", + env: &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{foundryEnvKey: "", azureAiEnvKey: "https://b"}, + }, + wantValue: "https://b", + wantName: "dev", + }, + { + name: "neither key is set", + env: &fakeEnv{current: envNamed("dev")}, + }, + { + // A key that is simply absent must not stop the second one being + // tried, nor the levels below. + name: "the first key is absent", + env: &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{azureAiEnvKey: "https://b"}, + valueErr: map[string]error{foundryEnvKey: status.Error(codes.NotFound, "no such key")}, + }, + wantValue: "https://b", + wantName: "dev", + }, + { + name: "reading a key failed", + env: &fakeEnv{ + current: envNamed("dev"), + valueErr: map[string]error{foundryEnvKey: status.Error(codes.Internal, "boom")}, + }, + wantErr: "boom", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + value, name, err := readEnvHostedSource(context.Background(), tc.env) + + if tc.wantErr != "" { + require.Error(t, err, "a failure to answer must stop the cascade") + assert.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err, "an answer of nothing must let the cascade carry on") + assert.Equal(t, tc.wantValue, value) + assert.Equal(t, tc.wantName, name) + }) + } +} + +// The newer key wins, so a project carrying both does not silently prefer the +// one an older command wrote. +func TestReadEnvHostedSource_PrefersTheNewerKey(t *testing.T) { + env := &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{foundryEnvKey: "https://new", azureAiEnvKey: "https://old"}, + } + + value, _, err := readEnvHostedSource(context.Background(), env) + + require.NoError(t, err) + assert.Equal(t, "https://new", value) + assert.Equal(t, []string{foundryEnvKey}, env.asked, + "the older key is not even read once the newer one answers") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go new file mode 100644 index 00000000000..7220ab1496f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// azd answers "no default environment" and "no such environment" with plain Go +// errors. Its interceptor only rewrites errors carrying a suggestion or an auth +// failure, so everything else reaches the client as Unknown. Reading Unknown as +// a failure would stop a project with no environment selected from ever +// reaching the global config or the host variable. +func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { + for name, err := range map[string]error{ + "no daemon at all": status.Error(codes.Unavailable, "connection refused"), + "nothing under that key": status.Error(codes.NotFound, "key not found"), + "no default environment": status.Error(codes.Unknown, "default environment not found"), + "no such environment": status.Error(codes.Unknown, "'dev': environment not found"), + // The atomic commands are meant to work standalone against the data + // plane with FOUNDRY_PROJECT_ENDPOINT exported, so running outside a + // project has to reach the host variable rather than stop here. + "outside a project": status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`"), + "wrapped in context": fmt.Errorf("reading the environment: %w", + status.Error(codes.Unknown, "default environment not found")), + } { + t.Run(name, func(t *testing.T) { + assert.True(t, hostedSourceAbsent(err), + "this is absence, so levels 3 and 4 still have to be consulted") + }) + } +} + +// A daemon that refused, or one that broke, is not a daemon with nothing to +// say. Falling through here would resolve to a lower-priority endpoint that can +// belong to a different project, and nothing would have said so. +func TestAFailureToAnswerIsReportedRatherThanSkipped(t *testing.T) { + for name, err := range map[string]error{ + "the login has expired": status.Error(codes.Unauthenticated, "the login has expired"), + "not allowed": status.Error(codes.PermissionDenied, "forbidden"), + "the user hit ctrl-c": status.Error(codes.Canceled, "context canceled"), + "the read timed out": status.Error(codes.DeadlineExceeded, "deadline exceeded"), + "the daemon broke": status.Error(codes.Internal, "internal error"), + "the daemon is full": status.Error(codes.ResourceExhausted, "quota exceeded"), + "the answer was corrupt": status.Error(codes.DataLoss, "data loss"), + // A bare error carries no status at all, so it never travelled the wire + // as an absence the daemon reported. + "not a status at all": errors.New("something local went wrong"), + // Unknown is not absence on its own. azd passes any error carrying no + // suggestion and no auth failure through untouched, so a failure to + // load project state or the environment manager arrives under the same + // code as "no default environment". + "project state would not load": status.Error(codes.Unknown, + "loading project state: open azure.yaml: permission denied"), + "the environment manager broke": status.Error(codes.Unknown, + "creating environment manager: no such host"), + // The message is the only evidence, so it is matched whole. A failure + // whose prose happens to mention one must not read as an absence. + "a failure that mentions an environment": status.Error(codes.Unknown, + "listing deployments: the environment not found in the subscription cache"), + // status.FromError flattens a wrapper's own prose into the message it + // reports, so a wrapper worded like an absence must not decide this. + "a failure wrapped in absence-sounding prose": fmt.Errorf( + "default environment not found in the cache: %w", + status.Error(codes.Unknown, "loading project state: permission denied")), + "wrapped expiry": fmt.Errorf("reading the environment: %w", + status.Error(codes.Unauthenticated, "expired")), + "nested twice over": fmt.Errorf("outer: %w", + fmt.Errorf("inner: %w", status.Error(codes.PermissionDenied, "no"))), + } { + t.Run(name, func(t *testing.T) { + assert.False(t, hostedSourceAbsent(err), + "a failure to answer has to surface, not resolve to a different project") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go new file mode 100644 index 00000000000..99827dc9160 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -0,0 +1,382 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ReadAzdHostedSourcesFunc is a package-level seam so tests can stub the +// daemon-backed lookup without spinning up a real azd gRPC server. +var ReadAzdHostedSourcesFunc = readAzdHostedSources + +// readAzdHostedSources dials the azd daemon (if reachable) and reads both the +// active environment's project endpoint and the global-config project context +// in a single client lifetime. The active-env read prefers +// FOUNDRY_PROJECT_ENDPOINT and falls back to AZURE_AI_PROJECT_ENDPOINT (the key +// `azd ai agent init` / `azd add` persist). Errors talking to the daemon are +// returned only for non-Unavailable cases on the config read — Unavailable is +// treated as "no daemon" and the caller falls through to subsequent levels. +func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { + var out AzdHostedSources + + azdClient, err := azdext.NewAzdClient() + if err != nil { + // No azd client at all => no hosted sources, not an error. + return out, nil + } + defer azdClient.Close() + + envValue, envName, envErr := readEnvHostedSource(ctx, azdClient.Environment()) + if envErr != nil { + return out, envErr + } + out.EnvValue, out.EnvName = envValue, envName + + state, found, cfgErr := getProjectContext(ctx, azdClient) + if cfgErr != nil { + // The same rule the environment reads use. Today the config service can + // only fail here by being unreachable, but stating it differently in + // one of three places is how the three come to disagree. + if !hostedSourceAbsent(cfgErr) { + return out, cfgErr + } + } else { + out.CfgState = state + out.CfgFound = found + } + + return out, nil +} + +// envSource is the slice of azd's environment service this file reads. +// +// Narrowed to an interface so the classification below can be tested. The rule +// it applies -- carry on when the daemon answered "nothing", stop when it +// failed to answer -- has regressed twice while every test passed, because the +// only seam was the whole function. +type envSource interface { + GetCurrent(context.Context, *azdext.EmptyRequest, ...grpc.CallOption) (*azdext.EnvironmentResponse, error) + GetValue(context.Context, *azdext.GetEnvRequest, ...grpc.CallOption) (*azdext.KeyValueResponse, error) +} + +// readEnvHostedSource reads the active environment's project endpoint. +// +// Returns an empty value and no error when there is nothing to read: no +// environment selected, no project at all, or neither key set. An error means +// the daemon failed to answer, which the caller must not read as absence -- +// falling through would resolve to a lower-priority endpoint that can belong to +// a different project. +// +// The environment is the one -e/--environment named, when it named one. Asking +// azd for the current environment instead is how `azd -e staging` came to read +// the endpoint out of the default environment and write its ids back there. +func readEnvHostedSource(ctx context.Context, env envSource) (value, name string, err error) { + selected := SelectedEnvironment(ctx) + name = selected + if name == "" { + envResp, envErr := env.GetCurrent(ctx, &azdext.EmptyRequest{}) + if envErr != nil { + if !hostedSourceAbsent(envErr) { + return "", "", envErr + } + return "", "", nil + } + if envResp.GetEnvironment() == nil { + return "", "", nil + } + name = envResp.Environment.Name + } + + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal, valErr := env.GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: name, + Key: key, + }) + if valErr != nil { + // A name the caller typed and azd does not have is a mistake to + // report, not an absence to step over. Falling through would run + // the command against a lower-priority endpoint -- possibly another + // project -- and then write its ids into an environment that does + // not exist, which azd accepts only far enough to warn about. + if selected != "" && noSuchEnvironment(valErr) { + return "", "", ErrNoSuchEnvironment(name) + } + if !hostedSourceAbsent(valErr) { + return "", "", valErr + } + continue + } + if envVal.GetValue() != "" { + return envVal.Value, name, nil + } + } + // The name is reported only alongside a value: it says where the endpoint + // came from, and there is no endpoint here. + return "", "", nil +} + +// noSuchEnvironment is azd's answer for a named environment it does not have, +// as distinct from the other absences: there being no default, or no project. +func noSuchEnvironment(err error) bool { + st, ok := azdext.GRPCStatusFromError(err) + if !ok || st.Code() != codes.Unknown { + return false + } + return strings.HasSuffix(st.Message(), "': "+azdNoSuchEnvironment) +} + +// ErrNoSuchEnvironment reports a -e/--environment naming something azd does not +// have. Built here rather than in either extension's messages package, so this +// file stays free of module-local imports and identical in both. +func ErrNoSuchEnvironment(name string) error { + return fmt.Errorf( + "azd environment %q does not exist; run `azd env list` to see the ones that do", + name, + ) +} + +// selectedEnvKey carries the environment -e/--environment named. +type selectedEnvKey struct{} + +// WithSelectedEnvironment records the environment the caller named, so every +// azd read and write in this invocation acts on that one rather than on azd's +// default. +// +// It travels on the context because the answer is fixed for the whole +// invocation and is needed several layers below the flag -- including here, in +// the cascade both extensions share, which has no cobra command to ask. +func WithSelectedEnvironment(ctx context.Context, name string) context.Context { + if name == "" { + return ctx + } + return context.WithValue(ctx, selectedEnvKey{}, name) +} + +// SelectedEnvironment is the name -e/--environment gave, or empty when it gave +// none and azd's default is what to act on. +func SelectedEnvironment(ctx context.Context) string { + name, _ := ctx.Value(selectedEnvKey{}).(string) + return name +} + +// envLookup is the one call needed to confirm a named environment exists. +type envLookup interface { + Get( + context.Context, *azdext.GetEnvironmentRequest, ...grpc.CallOption, + ) (*azdext.EnvironmentResponse, error) +} + +// VerifySelectedEnvironment refuses a -e/--environment azd does not have. +// +// Checked here rather than as a side effect of reading the endpoint, because +// the endpoint may not be read at all: --project-endpoint answers at level 1 +// and the cascade never runs, so `run start -e typo --project-endpoint ...` was +// accepted while the same command without the flag was refused. The name +// decides which environment every id, version and fingerprint is read from and +// written to, whichever level supplied the endpoint. +func VerifySelectedEnvironment(ctx context.Context) error { + name := SelectedEnvironment(ctx) + if name == "" { + return nil + } + client, err := azdext.NewAzdClient() + if err != nil { + // Nothing to ask: the extension is running outside azd. + return nil + } + defer client.Close() + + return verifyEnvironment(ctx, client.Environment(), name) +} + +// verifyEnvironment is the rule on its own, so it can be tested without a +// daemon. +// +// Only azd saying it has no such environment is an answer. Any other failure is +// not one, and is left to the commands that actually need azd to report, rather +// than turning a hiccup into "your environment does not exist". +func verifyEnvironment(ctx context.Context, env envLookup, name string) error { + _, err := env.Get(ctx, &azdext.GetEnvironmentRequest{Name: name}) + if err == nil { + return nil + } + if noSuchEnvironment(err) || containsGRPCCode(err, codes.NotFound) { + return ErrNoSuchEnvironment(name) + } + return nil +} + +// azd's absence sentinels, as they reach us. +// +// `pkg/environment` and `pkg/environment/azdcontext` declare these with +// errors.New, and the daemon's error-wrapping interceptor passes an error +// carrying no suggestion and no auth failure through untouched, so all three +// arrive as Unknown -- the same code a failure to load project state arrives +// under. The message is the only thing left to tell them apart. +// +// Matched whole rather than by substring. The message is the only evidence +// there is, so a failure whose prose happens to mention an environment must not +// read as one of these. The default-environment and no-project sentinels arrive +// on their own; the named-environment one arrives from the data store as +// `'': environment not found`. +// +// Matched rather than imported: taking a dependency on the environment manager +// for three strings costs more than it settles, and a rename fails closed. The +// command would report the daemon error instead of resolving quietly to a +// lower-priority endpoint, which is the direction to fail in. +const ( + azdNoDefaultEnvironment = "default environment not found" + azdNoSuchEnvironment = "environment not found" + azdNoProject = "no project exists; to create a new project, run `azd init`" +) + +// HostedSourceAbsent reports whether an error from the azd daemon is an answer +// of "nothing here" rather than a failure to answer. +// +// Exported because more than the cascade has to ask it. Deriving the set of +// azd's absences a second time elsewhere is how a sentinel comes to be handled +// in one place and missed in another, which has happened three times. +func HostedSourceAbsent(err error) bool { + return hostedSourceAbsent(err) +} + +// DaemonUnreachable reports the one absence that is not an answer about +// anything: there was nobody to ask. +// +// The cascade carries on regardless -- an unreachable daemon has no endpoint to +// offer, so the next level should be consulted. A caller reporting *why* a +// value is missing has to tell it apart, or a gRPC hiccup ends up phrased as a +// fact about the project. +func DaemonUnreachable(err error) bool { + return containsGRPCCode(err, codes.Unavailable) +} + +// hostedSourceAbsent reports whether an error from the azd daemon leaves the +// cascade free to carry on to the next level. +// +// Unavailable is no daemon at all. NotFound is a daemon with nothing under that +// name -- kept as a guard, though azd's environment service does not use it +// today. Unknown is the one that is not obvious: azd answers the ordinary +// absences with plain Go errors that reach us with no status, and without +// letting those through, a project with no environment selected -- or a command +// run outside a project at all, which the atomic commands are meant to support +// -- could never reach the global config or the host variable. It is admitted +// only for the three messages above, because Unknown is equally what a failure +// to load project state arrives as. +// +// Everything else is a failure to answer rather than an answer of "nothing": +// an expired login, a denial, a cancellation, or a server fault. Falling +// through on any of those would resolve quietly to a lower-priority endpoint +// that can belong to a different project. +func hostedSourceAbsent(err error) bool { + if containsGRPCCode(err, codes.Unavailable) || containsGRPCCode(err, codes.NotFound) { + return true + } + // The status the daemon sent, not the flattened text: status.FromError + // replaces a wrapped error's message with the whole of err.Error(), so the + // wrapper's own prose would take part in the comparison below. + st, ok := azdext.GRPCStatusFromError(err) + if !ok || st.Code() != codes.Unknown { + return false + } + msg := st.Message() + return msg == azdNoDefaultEnvironment || + msg == azdNoProject || + strings.HasSuffix(msg, "': "+azdNoSuchEnvironment) +} + +// containsGRPCCode walks the error chain looking for a gRPC status with the +// specified code. fmt.Errorf("%w", ...) wraps errors without forwarding the +// GRPCStatus() method, so we must unwrap manually. +// +// Note: only follows errors.Unwrap chains; errors.Join multi-wraps are not traversed. +func containsGRPCCode(err error, code codes.Code) bool { + for ; err != nil; err = errors.Unwrap(err) { + if st, ok := status.FromError(err); ok && st.Code() == code { + return true + } + } + return false +} + +// Resolve resolves a Foundry project endpoint using the 5-level cascade: +// +// 1. --project-endpoint flag +// 2. Active azd env value (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. Global config: extensions.ai-agents.project.context.endpoint (read-only; +// owned by azure.ai.agents) +// 4. Host environment variable (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 5. Structured error with actionable suggestion +// +// Invalid values at any level produce a hard validation error (no silent fallback). +func Resolve(ctx context.Context, opts ResolveOpts) (*Resolved, error) { + // Level 1: explicit flag. + if opts.FlagValue != "" { + normalized, _, err := Validate(opts.FlagValue) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFlag}, nil + } + + // Levels 2 + 3: azd-hosted sources (active env, then global config). + sources, err := ReadAzdHostedSourcesFunc(ctx) + if err != nil { + return nil, err + } + + // Level 2: active azd environment's FOUNDRY_PROJECT_ENDPOINT (with the + // AZURE_AI_PROJECT_ENDPOINT fallback applied in readAzdHostedSources). + if sources.EnvValue != "" { + normalized, _, err := Validate(sources.EnvValue) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceAzdEnv, + AzdEnvName: sources.EnvName, + }, nil + } + + // Level 3: global config (~/.azd/config.json). + if sources.CfgFound && sources.CfgState.Endpoint != "" { + normalized, _, err := Validate(sources.CfgState.Endpoint) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceGlobalConfig, + SetAt: sources.CfgState.SetAt, + }, nil + } + + // Level 4: host environment variable (FOUNDRY_PROJECT_ENDPOINT, then the + // AZURE_AI_PROJECT_ENDPOINT fallback). + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal := os.Getenv(key) + if envVal == "" { + continue + } + normalized, _, err := Validate(envVal) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFoundryEnv}, nil + } + + // Level 5: structured error. + return nil, NoEndpointError() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go new file mode 100644 index 00000000000..f5300db6fc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "testing" + + "azureaieval/internal/exterrors" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withHostedSources installs a stub for ReadAzdHostedSourcesFunc for the +// duration of the test and restores the production value on cleanup. Tests +// using this MUST NOT run in parallel because the seam is a package-level var. +func withHostedSources(t *testing.T, sources AzdHostedSources, err error) { + t.Helper() + orig := ReadAzdHostedSourcesFunc + ReadAzdHostedSourcesFunc = func(context.Context) (AzdHostedSources, error) { + return sources, err + } + t.Cleanup(func() { ReadAzdHostedSourcesFunc = orig }) +} + +// isolateFromAzdDaemon installs an empty hosted-sources stub and clears +// AZD_SERVER so any code path that bypasses the seam cannot reach a real +// daemon. After calling this, the resolver only sees the flag and the +// FOUNDRY_PROJECT_ENDPOINT / AZURE_AI_PROJECT_ENDPOINT host env vars. +func isolateFromAzdDaemon(t *testing.T) { + t.Helper() + t.Setenv("AZD_SERVER", "") + withHostedSources(t, AzdHostedSources{}, nil) +} + +func TestResolve_FlagWins(t *testing.T) { + // Even with FOUNDRY_PROJECT_ENDPOINT and azd-hosted sources set, the flag wins. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + withHostedSources(t, AzdHostedSources{ + EnvValue: "https://azdenv.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "https://flag.services.ai.azure.com/api/projects/flag-proj", + }) + require.NoError(t, err) + assert.Equal(t, "https://flag.services.ai.azure.com/api/projects/flag-proj", result.Endpoint) + assert.Equal(t, SourceFlag, result.Source) +} + +func TestResolve_AzdEnvWinsOverConfigAndFoundryEnv(t *testing.T) { + // EnvValue here stands in for whichever active-env key readAzdHostedSources + // resolved (FOUNDRY_PROJECT_ENDPOINT, or the AZURE_AI_PROJECT_ENDPOINT + // fallback); either way level 2 wins over global config and the host env. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: " HTTPS://Azdenv.Services.AI.Azure.com/api/projects/p/ ", + EnvName: "dev", + CfgState: State{ + Endpoint: "https://cfg.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-01T00:00:00Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azdenv.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceAzdEnv, result.Source) + assert.Equal(t, "dev", result.AzdEnvName) +} + +func TestResolve_AzdEnvInvalidIsHardError(t *testing.T) { + // Level 2 invalid values are hard errors (no silent fallback to lower levels). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: "http://not-https.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_GlobalConfigWinsOverFoundryEnv(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: " HTTPS://Cfg.Services.AI.Azure.com/api/projects/p/ ", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://cfg.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceGlobalConfig, result.Source) + assert.Equal(t, "2025-01-02T03:04:05Z", result.SetAt) +} + +func TestResolve_GlobalConfigInvalidIsHardError(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: "http://not-https.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_HostedSourcesErrorPropagates(t *testing.T) { + // Non-recoverable errors from the hosted-source lookup must be surfaced + // and must not silently fall through to level 4. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + sentinel := errors.New("boom") + withHostedSources(t, AzdHostedSources{}, sentinel) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.ErrorIs(t, err, sentinel) +} + +func TestResolve_FoundryEnvFallback(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/env-proj", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_AzureAiHostEnvFallback(t *testing.T) { + // When FOUNDRY_PROJECT_ENDPOINT is unset, the resolver falls back to the + // AZURE_AI_PROJECT_ENDPOINT host env var (the key azd ai agent init / azd + // add persist). See https://github.com/Azure/azure-dev/issues/8688. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/p") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azureai.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryHostEnvWinsOverAzureAi(t *testing.T) { + // With both host env vars set, FOUNDRY_PROJECT_ENDPOINT takes precedence. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/f") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/a") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://foundry.services.ai.azure.com/api/projects/f", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryEnvNormalized(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", " https://X.SERVICES.AI.AZURE.COM/api/projects/p/ ") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://x.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_InvalidFlagRejected(t *testing.T) { + isolateFromAzdDaemon(t) + + _, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "http://not-https.services.ai.azure.com/api/projects/p", + }) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidFoundryEnvRejected(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "http://bad.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidAzureAiHostEnvRejected(t *testing.T) { + // An invalid AZURE_AI_PROJECT_ENDPOINT fallback is a hard error, not a + // silent skip to level 5. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "http://not-https.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_NothingResolvable(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, exterrors.CodeMissingProjectEndpoint, localErr.Code) + assert.Equal(t, azdext.LocalErrorCategoryDependency, localErr.Category) +} + +func TestResolve_CfgFoundButEndpointEmptyFallsThrough(t *testing.T) { + // CfgFound=true with Endpoint="" must not short-circuit; the resolver + // should continue to level 4 (FOUNDRY_PROJECT_ENDPOINT). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{Endpoint: "", SetAt: "2025-01-01T00:00:00Z"}, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestContainsGRPCCode_NonGRPCErrorReturnsFalse(t *testing.T) { + t.Parallel() + assert.False(t, containsGRPCCode(errors.New("plain"), 0)) + assert.False(t, containsGRPCCode(nil, 0)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go new file mode 100644 index 00000000000..57704a20507 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// -e/--environment is parsed by the SDK and then has to be acted on. It was +// discarded, so `azd ai eval create -e staging` read its endpoint out of the +// default environment and wrote its ids back there -- and `-e a-name-azd- +// rejects` was accepted in silence, because nothing ever asked azd about it. +// +// These tests pin that the named environment is the one read, and that azd is +// not asked which environment is current when a name was given: asking can only +// produce a second, disagreeing answer. + +// perEnv answers GetValue per environment, which is what tells "read staging" +// apart from "read whatever azd calls current". +type perEnv struct { + values map[string]map[string]string + current string + currentCalls int + askedEnvs []string +} + +func (f *perEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + f.currentCalls++ + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: f.current}, + }, nil +} + +func (f *perEnv) GetValue( + _ context.Context, req *azdext.GetEnvRequest, _ ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + f.askedEnvs = append(f.askedEnvs, req.EnvName) + return &azdext.KeyValueResponse{ + Key: req.Key, + Value: f.values[req.EnvName][req.Key], + }, nil +} + +func twoEnvironments() *perEnv { + return &perEnv{ + values: map[string]map[string]string{ + "default": {foundryEnvKey: "https://from-default/"}, + "staging": {foundryEnvKey: "https://from-staging/"}, + }, + current: "default", + } +} + +func TestSelectedEnvironmentIsTheOneRead(t *testing.T) { + fake := twoEnvironments() + + ctx := WithSelectedEnvironment(context.Background(), "staging") + value, name, err := readEnvHostedSource(ctx, fake) + + require.NoError(t, err) + assert.Equal(t, "https://from-staging/", value) + assert.Equal(t, "staging", name) + assert.Zero(t, fake.currentCalls, + "a named environment is the answer; asking azd for the current one can only disagree") + assert.NotContains(t, fake.askedEnvs, "default") +} + +func TestWithoutSelectionAzdsCurrentEnvironmentIsRead(t *testing.T) { + fake := twoEnvironments() + + value, name, err := readEnvHostedSource(context.Background(), fake) + + require.NoError(t, err) + assert.Equal(t, "https://from-default/", value) + assert.Equal(t, "default", name) + assert.Equal(t, 1, fake.currentCalls) +} + +// A named environment holding no endpoint reports none. Falling back to the +// default's is the bug, restated. +func TestSelectedEnvironmentWithNoEndpointDoesNotFallBack(t *testing.T) { + fake := twoEnvironments() + fake.values["staging"] = map[string]string{} + + ctx := WithSelectedEnvironment(context.Background(), "staging") + value, _, err := readEnvHostedSource(ctx, fake) + + require.NoError(t, err) + assert.Empty(t, value, "staging has no endpoint; the default's is not an answer") + assert.Zero(t, fake.currentCalls) + assert.NotContains(t, fake.askedEnvs, "default") +} + +// An empty name is "none given", not "the environment called empty string". +func TestWithSelectedEnvironmentIgnoresAnEmptyName(t *testing.T) { + assert.Empty(t, SelectedEnvironment(WithSelectedEnvironment(context.Background(), ""))) + assert.Equal(t, "staging", + SelectedEnvironment(WithSelectedEnvironment(context.Background(), "staging"))) + assert.Empty(t, SelectedEnvironment(context.Background())) +} + +// failingEnv answers GetValue with a fixed error, which is how azd reports an +// environment it does not have. +type failingEnv struct { + err error + currentCalls int +} + +func (f *failingEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + f.currentCalls++ + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: "default"}, + }, nil +} + +func (f *failingEnv) GetValue( + context.Context, *azdext.GetEnvRequest, ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + return nil, f.err +} + +// A name the caller typed and azd does not have is a mistake to report, not an +// absence to step over. Stepping over it runs the command against a +// lower-priority endpoint, which can belong to another project, and then writes +// its ids into an environment that does not exist. +func TestATypoedEnvironmentNameIsReportedNotSteppedOver(t *testing.T) { + fake := &failingEnv{ + err: status.Error(codes.Unknown, "'does-not-exist': environment not found"), + } + + ctx := WithSelectedEnvironment(context.Background(), "does-not-exist") + _, _, err := readEnvHostedSource(ctx, fake) + + require.Error(t, err, "a named environment azd does not have must stop the cascade") + assert.Contains(t, err.Error(), "does-not-exist") +} + +// The same answer without a name given is ordinary absence: there is simply no +// endpoint in the current environment, and the cascade carries on. +func TestTheSameAnswerWithoutANameIsStillAbsence(t *testing.T) { + fake := &failingEnv{ + err: status.Error(codes.Unknown, "'default': environment not found"), + } + + value, name, err := readEnvHostedSource(context.Background(), fake) + + require.NoError(t, err, "without -e this is absence, and the cascade continues") + assert.Empty(t, value) + assert.Empty(t, name) +} + +// A named environment that exists but cannot be read for some other reason is +// a failure, and must not be reported as a missing environment either. +func TestANamedEnvironmentThatFailsDifferentlyStillFails(t *testing.T) { + fake := &failingEnv{err: status.Error(codes.Internal, "the store is on fire")} + + ctx := WithSelectedEnvironment(context.Background(), "staging") + _, _, err := readEnvHostedSource(ctx, fake) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "does not exist", + "a broken read is not a missing environment") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go new file mode 100644 index 00000000000..dd899fdee9f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectContextConfigPath is the read-only UserConfig path for the persisted +// project context owned by azure.ai.agents. The toolboxes extension reads this +// key but never writes it (§ 6 of the design spec). +const projectContextConfigPath = "extensions.ai-agents.project.context" + +// getProjectContext reads the persisted project context from global config. +// Returns (state, true, nil) when present, (zero, false, nil) when absent. +func getProjectContext( + ctx context.Context, azdClient *azdext.AzdClient, +) (State, bool, error) { + ch, err := azdext.NewConfigHelper(azdClient) + if err != nil { + return State{}, false, messages.ProjectContextClient(err) + } + + var state State + found, err := ch.GetUserJSON(ctx, projectContextConfigPath, &state) + if err != nil { + return State{}, false, messages.ProjectContextRead(err) + } + + if !found || state.Endpoint == "" { + return State{}, false, nil + } + + return state, true, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go new file mode 100644 index 00000000000..93bf43f3780 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package projectctx encapsulates the Foundry project endpoint cascade and +// validation shared by every Foundry-extension command tree. +// +// This is the toolboxes-extension copy of the agent_context.go / project_endpoint.go / +// project_context_store.go logic in azure.ai.agents (see § 3.2 of the toolbox +// design spec). Semantics match the agents original verbatim; identifiers are +// exported because they cross the package boundary in this layout. +package projectctx + +const ( + // foundryEnvKey is the canonical project-endpoint key. It is read both from + // the active azd environment (level 2) and as a host environment variable + // (level 4). + foundryEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + // azureAiEnvKey is the legacy/sibling project-endpoint key written by + // `azd ai agent init` and `azd add` (Bicep output). It is read as a fallback + // after foundryEnvKey at both the active-azd-env and host-env levels so the + // hosted-agent + toolbox workflow resolves without an extra manual step. + // See https://github.com/Azure/azure-dev/issues/8688. + azureAiEnvKey = "AZURE_AI_PROJECT_ENDPOINT" +) + +// EndpointSource identifies where a resolved project endpoint came from. +type EndpointSource string + +const ( + // SourceFlag means the endpoint came from the --project-endpoint flag. + SourceFlag EndpointSource = "flag" + // SourceAzdEnv means the endpoint came from the active azd environment's + // FOUNDRY_PROJECT_ENDPOINT (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) value. + SourceAzdEnv EndpointSource = "azdEnv" + // SourceGlobalConfig means the endpoint came from ~/.azd/config.json + // (extensions.ai-agents.project.context.endpoint — owned by azure.ai.agents + // and shared read-only with sibling extensions). + SourceGlobalConfig EndpointSource = "globalConfig" + // SourceFoundryEnv means the endpoint came from the FOUNDRY_PROJECT_ENDPOINT + // (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) host environment variable. + SourceFoundryEnv EndpointSource = "foundryEnv" +) + +// ResolveOpts controls the 5-level endpoint resolution cascade. +type ResolveOpts struct { + // FlagValue is the value of the --project-endpoint flag (level 1). + // Empty means the flag was not provided. + FlagValue string +} + +// Resolved holds the result of Resolve. +type Resolved struct { + Endpoint string + Source EndpointSource + AzdEnvName string + SetAt string // RFC3339 timestamp; only meaningful when Source == SourceGlobalConfig +} + +// AzdHostedSources holds the values the resolver reads from azd-managed +// sources (active env + ~/.azd/config.json). Returned as a single struct so +// tests can stub the whole lookup via ReadAzdHostedSourcesFunc. +type AzdHostedSources struct { + // EnvValue is the active-azd-env project endpoint: FOUNDRY_PROJECT_ENDPOINT + // if set, otherwise AZURE_AI_PROJECT_ENDPOINT, otherwise "" (not set / no + // active env / no azd client available). + EnvValue string + // EnvName is the active azd env name. Only meaningful when EnvValue != "". + EnvName string + // CfgState is the project context persisted in global config. + CfgState State + // CfgFound indicates whether a non-empty endpoint was found in global config. + CfgFound bool +} + +// State is the JSON shape stored at extensions.ai-agents.project.context in +// ~/.azd/config.json. This key is owned by azure.ai.agents; the toolboxes +// extension reads it but never writes it. +type State struct { + Endpoint string `json:"endpoint"` + SetAt string `json:"setAt"` +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go new file mode 100644 index 00000000000..9d2de6a5d95 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "fmt" + "net/url" + "strings" + + "azureaieval/internal/messages" +) + +// foundryHostSuffixes is the authoritative list of accepted Foundry host suffixes. +var foundryHostSuffixes = []string{ + ".services.ai.azure.com", +} + +// projectEndpointPathPrefix is the expected path prefix for Foundry project endpoints. +const projectEndpointPathPrefix = "/api/projects/" + +// isFoundryHost reports whether the hostname ends with a recognized Foundry suffix. +func isFoundryHost(hostname string) bool { + h := strings.ToLower(hostname) + for _, suffix := range foundryHostSuffixes { + if strings.HasSuffix(h, suffix) { + return true + } + } + return false +} + +// Validate validates and normalizes a Foundry project endpoint URL. +// +// The URL must be an absolute https:// URL whose host ends with a recognized +// Foundry suffix. Whitespace is trimmed, trailing slashes are stripped, and +// the result is returned in normalized form. +// +// The second return value is true when the path does not look like +// /api/projects/ — callers may use this as a non-fatal warning. +func Validate(raw string) (normalized string, pathWarning bool, err error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", false, messages.EndpointEmpty() + } + + u, parseErr := url.Parse(raw) + if parseErr != nil { + return "", false, messages.EndpointUnparseable(parseErr) + } + + if !strings.EqualFold(u.Scheme, "https") { + return "", false, messages.EndpointNotHTTPS() + } + + host := u.Hostname() + if host == "" || !isFoundryHost(host) { + return "", false, messages.EndpointNotFoundryHost(host, foundryHostSuffixes[0]) + } + + if u.Port() != "" { + return "", false, messages.EndpointHasPort(u.Host) + } + + // Normalize: lowercase host, strip trailing slash. + path := strings.TrimRight(u.EscapedPath(), "/") + normalized = fmt.Sprintf("https://%s%s", strings.ToLower(host), path) + + // Warn when the path does not look like /api/projects/. + if !strings.HasPrefix(path, projectEndpointPathPrefix) || + strings.TrimPrefix(path, projectEndpointPathPrefix) == "" { + pathWarning = true + } + + return normalized, pathWarning, nil +} + +// NoEndpointError returns the structured dependency error used when no project +// endpoint could be resolved from any source. +func NoEndpointError() error { + return messages.NoEndpoint() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go new file mode 100644 index 00000000000..81b866a9020 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// lookupStub answers the one call that confirms an environment exists. +type lookupStub struct { + err error + asked []string +} + +func (l *lookupStub) Get( + _ context.Context, req *azdext.GetEnvironmentRequest, _ ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + l.asked = append(l.asked, req.Name) + if l.err != nil { + return nil, l.err + } + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: req.Name}, + }, nil +} + +// The named-environment check used to live inside the endpoint cascade, which +// --project-endpoint skips entirely: `run start -e typo --project-endpoint ...` +// was accepted while the same command without the flag was refused. The name +// decides where every id and version is read from and written to, whichever +// level supplied the endpoint, so the check does not belong to any level. +func TestAnEnvironmentAzdDoesNotHaveIsRefused(t *testing.T) { + stub := &lookupStub{ + err: status.Error(codes.Unknown, "'typo': environment not found"), + } + + err := verifyEnvironment(context.Background(), stub, "typo") + + require.Error(t, err) + assert.Contains(t, err.Error(), "typo") + assert.Equal(t, []string{"typo"}, stub.asked) +} + +func TestAnEnvironmentAzdHasIsAccepted(t *testing.T) { + stub := &lookupStub{} + + require.NoError(t, verifyEnvironment(context.Background(), stub, "staging")) + assert.Equal(t, []string{"staging"}, stub.asked) +} + +// A daemon that could not answer has not said the environment is missing. +// Refusing there would turn a hiccup into "your environment does not exist"; +// the commands that need azd report their own failures. +func TestAFailureThatIsNotAnAnswerDoesNotRefuse(t *testing.T) { + for _, err := range []error{ + status.Error(codes.Internal, "the store is on fire"), + status.Error(codes.Unavailable, "no daemon"), + status.Error(codes.Unknown, "no project exists; to create a new project, run `azd init`"), + } { + stub := &lookupStub{err: err} + assert.NoError(t, verifyEnvironment(context.Background(), stub, "staging"), + "unexpected refusal for %v", err) + } +} + +// Nothing named means azd's default, which needs no confirming. +func TestNoSelectionAsksNothing(t *testing.T) { + require.NoError(t, VerifySelectedEnvironment(context.Background())) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go new file mode 100644 index 00000000000..4ba33ca1b3f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "net/http" + "path/filepath" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/stretchr/testify/assert" +) + +func notFoundErr() error { + return &azcore.ResponseError{StatusCode: http.StatusNotFound, ErrorCode: "not_found"} +} + +// A misspelled --target is the common cause, and 404 answers it with ten lines +// of URL, status rule and nested JSON for a fact that fits on one. +func TestAgentWarningsAnswerA404InOneLine(t *testing.T) { + for _, line := range []string{ + WarningAgentUnreadable("suport-agent", notFoundErr()), + CouldNotReadAgentForModel("suport-agent", notFoundErr()), + } { + assert.Contains(t, line, `no agent "suport-agent" in this project`) + assert.NotContains(t, line, "RESPONSE 404") + assert.NotContains(t, line, "https://") + assert.Equal(t, 1, countLines(line), "a 404 is one fact, so it gets one line: %s", line) + } + + // Anything else keeps the detail, because it is not a fact anyone knows yet. + other := WarningAgentUnreadable("support-agent", errors.New("connection reset")) + assert.Contains(t, other, "connection reset") +} + +// The declared dataset has not been generated yet, which is an ordering mistake +// and not a broken configuration. The bare stat failure underneath is a Windows +// syscall name and says nothing about what to run. +func TestDatasetNotGeneratedYet(t *testing.T) { + err := DatasetProblem("support-agent-eval", + DatasetNotGeneratedYet("support-agent-eval", + filepath.Join("evals", "datasets", "support-agent-eval.jsonl"))) + + got := err.Error() + assert.Contains(t, got, "evals/datasets/support-agent-eval.jsonl") + assert.NotContains(t, got, `\\`) + assert.NotContains(t, got, "GetFileAttributesEx") + assert.Contains(t, got, "azd ai eval generate --dataset --dataset-name support-agent-eval") + assert.Equal(t, 1, countOccurrences(got, `"support-agent-eval"`), + "the wrapper names the dataset, so the message must not name it again") +} + +func countLines(s string) int { + n := 0 + for _, r := range s { + if r == '\n' { + n++ + } + } + return n +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go new file mode 100644 index 00000000000..5e31898de78 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "fmt" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// RequestFailed rewrites a credential failure into "run `azd auth login`". That +// is the right answer for a token that could not be minted and the wrong answer +// for anything else, so what counts as one has to be narrow. +func TestRequestFailedOnlyClaimsAuthForRealCredentialFailures(t *testing.T) { + t.Run("a credential failure is rewritten", func(t *testing.T) { + // What AzureDeveloperCLICredential returns when `azd auth token` exits + // non-zero: the shape seen live as "exit status 1". + err := RequestFailed(errors.New( + "AzureDeveloperCLICredential: exit status 1")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") + }) + + // The regression this test exists for. "failed to acquire a token" used to + // be matched anywhere in the text, so an unrelated failure that happened to + // contain the phrase was reported as an expired login. + t.Run("an unrelated error keeping that phrase is left alone", func(t *testing.T) { + err := RequestFailed(errors.New( + "the pool failed to acquire a token bucket lease")) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "a lease is not a login") + assert.Contains(t, err.Error(), "token bucket lease", + "and the original failure still has to be readable") + }) + + t.Run("an ordinary transport failure is passed through", func(t *testing.T) { + err := RequestFailed(errors.New("connection reset by peer")) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login") + assert.Contains(t, err.Error(), "connection reset by peer") + }) + + // Matching the SDK's type rather than its wording means a reworded message + // still classifies, and a lookalike string does not. + t.Run("the SDK's own type classifies whatever it says", func(t *testing.T) { + var typed error = &azidentity.AuthenticationFailedError{} + err := RequestFailed(fmt.Errorf("getting a token: %w", typed)) + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") + }) + + t.Run("nil stays nil-ish", func(t *testing.T) { + assert.False(t, isCredentialFailure(nil)) + }) +} + +// A credential that never ran is a different problem from one that ran and was +// refused, and `azd auth login` is not the answer to it -- you cannot log in +// with a tool that is not on PATH. +func TestRequestFailedSeparatesAnUnrunnableCredentialFromAnExpiredLogin(t *testing.T) { + for _, text := range []string{ + "AzureDeveloperCLICredential: executable not found on path", + "AzureDeveloperCLICredential: 'azd' is not recognized as an internal or external command", + } { + err := RequestFailed(errors.New(text)) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "cannot log in with a tool that will not run: %s", text) + assert.Contains(t, err.Error(), "could not be run") + } + + // The expired-login case must still say what fixes it. + err := RequestFailed(errors.New("AzureDeveloperCLICredential: exit status 1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") +} + +// A 401 or 403 is the service refusing a token it did read, which is a +// different fix from a token that was never minted. +func TestServiceRefusedOnlyRewritesUnauthorized(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := ServiceRefused(status, errors.New("nope")) + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login", "status %d", status) + } + + err := ServiceRefused(http.StatusInternalServerError, errors.New("boom")) + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "a 500 is not something a fresh login fixes") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go new file mode 100644 index 00000000000..2a0da8beedc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// An eval is immutable, so editing a declaration creates another under the same +// name and leaves the previous one holding its run history. Deleting takes the +// runs with it, so a name that matches several has to be refused rather than +// resolved to whichever happens to sort first. +func TestAmbiguousEvalName(t *testing.T) { + got := AmbiguousEvalName("support-agent-eval", []string{"eval_aaa", "eval_bbb"}).Error() + + assert.Contains(t, got, `2 evals are named "support-agent-eval"`) + assert.Contains(t, got, "eval_aaa") + assert.Contains(t, got, "eval_bbb") + assert.Contains(t, got, "discards its runs", "the reason it will not guess is the cost of guessing") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go new file mode 100644 index 00000000000..95783056a73 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The bare stat failure underneath this is a Windows syscall name and a path +// with doubled separators, which says nothing about what to do next. Both +// callers wrap it with EvaluatorProblem, so the evaluator is named once. +func TestEvaluatorNotGeneratedYet(t *testing.T) { + err := EvaluatorProblem("support-agent-quality", + EvaluatorNotGeneratedYet("support-agent-quality", + filepath.Join("evals", "evaluators", "support-agent-quality.json"))) + + got := err.Error() + assert.Equal(t, 1, countOccurrences(got, `"support-agent-quality"`), + "the wrapper names the evaluator, so the message must not name it again") + assert.Contains(t, got, "evals/evaluators/support-agent-quality.json", + "the path reads as a path, not as an escaped Windows literal") + assert.NotContains(t, got, `\\`) + assert.Contains(t, got, "azd ai eval generate --evaluator --evaluator-name support-agent-quality", + "the way out is the command that writes the definition") +} + +func countOccurrences(s, sub string) int { + n := 0 + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + n++ + } + } + return n +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go new file mode 100644 index 00000000000..326be1adcf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -0,0 +1,2691 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package messages holds every string this extension shows a user. +// +// One file, so the whole voice of the CLI can be reviewed in one sitting and a +// wording change never has to be hunted through the command tree. The only +// extension package it imports is exterrors, which holds no wording of its own, +// so every other package can use this one. +// +// Conventions, so the set stays consistent: +// +// - Errors state what went wrong and, where there is one, the way out. +// Lowercase, no trailing period: azd renders them after "ERROR: ". +// - A name the user chose is quoted with %q; an identifier the service +// assigned is not, because it is already unmistakable. +// - A filesystem path goes through filepath.ToSlash first. %q escapes a +// Windows separator, so `evals\eval.yaml` prints as "evals\\eval.yaml" and +// a reader who copies it back gets a path that does not exist. +// - Progress and success lines are sentences with a capital and no period. +// - A printed line carries its own newlines, so a call site is a bare Fprint. +// - Nothing here decides *whether* to print. That stays at the call site. +package messages + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "azureaieval/internal/exterrors" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// --------------------------------------------------------------------------- +// Running an eval +// --------------------------------------------------------------------------- + +// NoEvalToRun reports a run with nothing resolved to run. +func NoEvalToRun() error { + return errors.New("no eval to run") +} + +// EvalHasNoDataset reports an eval whose rows cannot be located. +// +// Named separately from the traces and responses cases because the way out is +// different: this one is answered by a dataset, not by a source block. +func EvalHasNoDataset(eval string) error { + return fmt.Errorf( + "eval %q references no dataset and declares no source. Add a dataset: to "+ + "score rows you supply, or a source: to score traces or stored responses", + eval) +} + +// DatasetFileEmpty reports a local dataset file that parsed but held no rows. +func DatasetFileEmpty(path string) error { + return fmt.Errorf("dataset file %q has no rows", filepath.ToSlash(path)) +} + +// DatasetOverrideNeedsDeclaredEval reports --dataset passed against a bare id. +func DatasetOverrideNeedsDeclaredEval() error { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") +} + +// DatasetNotInCatalog reports a --dataset the configuration does not declare. +func DatasetNotInCatalog(dataset, configPath string) error { + return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, filepath.ToSlash(configPath)) +} + +// NothingToGenerateFrom refuses a generation request carrying no sources. +// +// The service answers one with "At least one source is required", wrapped in a +// 400 and thirty lines of JSON. Nothing about that names the two things a +// reader can actually supply. +func NothingToGenerateFrom() error { + return errors.New( + "nothing to generate from: pass --target to seed from an " + + "agent's instructions, or declare one under target: in the eval " + + "configuration. A trace-backed eval names its agent under source:, " + + "which selects traces to read and does not seed generation") +} + +// SelectEvaluatorsPrompt asks which references the eval grades on. +func SelectEvaluatorsPrompt() string { + return "Select evaluators to grade with:" +} + +// SelectingEvaluators reports a failed evaluator prompt. +func SelectingEvaluators(err error) error { + return fmt.Errorf("selecting evaluators: %w", err) +} + +// NoEvaluatorsChosen reports an eval that would grade on nothing. +func NoEvaluatorsChosen() error { + return errors.New( + "an eval has to grade on at least one evaluator: select one, or pass " + + "--evaluator") +} + +// GateNeedsTheWait refuses a gate on a run the command will not wait for. +// +// The two flags together read as "start it and tell me if it regressed", but +// the verdict does not exist yet when --no-wait returns, so the gate was +// silently dropped and the command exited 0 however the run turned out. +func GateNeedsTheWait() error { + return errors.New( + "--fail-on needs a result to judge, and --no-wait returns before there " + + "is one. Drop --no-wait, or reattach with `azd ai eval run show " + + " --wait --fail-on `") +} + +// GateOutlivedTheWait reports a gate that never got a verdict because the run +// outlived the wait. +// +// Without a gate this is not a failure and exits 0, which is why the run was +// reported and the reattach line printed. With one, exiting 0 tells a pipeline +// the gate passed when nothing was ever judged -- the same silent drop +// GateNeedsTheWait refuses up front, arrived at by running long instead. +func GateOutlivedTheWait(runID string, budget time.Duration) error { + return fmt.Errorf( + "run %s outlived the %s wait, so --fail-on never got a result to judge. "+ + "The run is still going: reattach with `azd ai eval run show %s "+ + "--wait --fail-on `", runID, budget, runID) +} + +// DatasetHasUnregisteredEdits reports local rows no deployed version holds. +func DatasetHasUnregisteredEdits(dataset, deployCmd string) error { + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `%s` to register them, or `--eval ` to run against "+ + "an existing eval", + dataset, deployCmd) +} + +// StartingRun reports the service refusing to start the run. +func StartingRun(err error) error { + return fmt.Errorf("starting the evaluation run: %w", err) +} + +// RunStarted reports a submitted run that was not waited on. +func RunStarted(runID, status string) string { + return fmt.Sprintf("Started run %s (status: %s)\n", runID, status) +} + +// ReattachToRun says how to come back to a run started with --no-wait. +func ReattachToRun(runID, evalID string) string { + return fmt.Sprintf("Reattach with: azd ai eval run show %s --eval %s\n", runID, evalID) +} + +// ReadingPreviousRuns reports a failure to look up what an eval last ran. +func ReadingPreviousRuns(evalID string, err error) error { + return fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) +} + +// EvalHasNoPreviousRun reports an eval named by id that has nothing to repeat. +func EvalHasNoPreviousRun(evalID string) error { + return fmt.Errorf( + "eval %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", + evalID) +} + +// PollingRun reports a failure while waiting for a run to finish. +func PollingRun(runID string, err error) error { + return fmt.Errorf("polling run %s: %w", runID, err) +} + +// WaitBudgetSpent reports a run that outlived the foreground wait. +func WaitBudgetSpent(runID string, budget time.Duration) string { + return fmt.Sprintf( + "Run %s is still going after %s, so the wait stopped, not the run.\n", + runID, budget) +} + +// WaitInterrupted reports a wait cut short, naming the run still in flight. +func WaitInterrupted(runID string, err error) error { + return fmt.Errorf( + "stopped waiting on run %s, which is still running: %w. "+ + "Pick it back up with `azd ai eval run show %s`", + runID, err, runID) +} + +// RunStatusLine reports a status change seen while polling. +func RunStatusLine(status string) string { + return fmt.Sprintf(" status: %s\n", status) +} + +// RunFinishedWithStatus reports a run that ended in something other than completed. +func RunFinishedWithStatus(runID, status string) error { + return fmt.Errorf("run %s finished with status %s", runID, status) +} + +// OverallPassRate reports the share of the rows an evaluator scored that passed +// every evaluator. +// +// The denominator is named rather than left as a bare fraction. Rows nothing +// could grade are outside it, so a run that errored on most of its samples can +// report a high rate, and "of N scored" is what stops that reading as a verdict +// on the whole run. It is also the figure `--fail-on pass-rate` compares. +func OverallPassRate(rate string, passed, scored, unscored int) string { + if unscored > 0 { + return fmt.Sprintf("\nOverall pass rate: %s (%d of %d scored; %d not scored)\n", + rate, passed, scored, unscored) + } + return fmt.Sprintf("\nOverall pass rate: %s (%d/%d)\n", rate, passed, scored) +} + +// SamplesErrored reports rows the run could not score at all. +func SamplesErrored(errored int) string { + return fmt.Sprintf("%d sample(s) errored and were not scored.\n", errored) +} + +// ViewFailingSamples points at the command that lists the rows that failed. +func ViewFailingSamples() string { + return "\nView failing samples: azd ai eval run output list --failed-only\n" +} + +// ErroredNotScored annotates an evaluator's row with what it could not score. +func ErroredNotScored(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// ReportLink closes a run summary with the one link the run has. +func ReportLink(url string) string { + return fmt.Sprintf("Report: %s\n", url) +} + +// EvalNotDeployed reports an eval id the project does not hold. +func EvalNotDeployed(evalID, deployCmd string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`%s` creates the ones your config declares", evalID, deployCmd) +} + +// NoEnvironmentToRememberEval reports an eval whose id had nowhere to be kept. +// +// `create` publishes the eval and records its id in the azd environment. With +// no environment there is nowhere to record it, so create reports success and +// the next command cannot find what it made. Saying "not deployed" there sends +// the reader to deploy it again, which lands in the same place. +func NoEnvironmentToRememberEval(eval string) error { + return fmt.Errorf( + "eval %q may exist in the project, but this directory has no azd "+ + "environment to have recorded its id in. Create one with "+ + "`azd env new ` and run `azd ai eval create` again, or name "+ + "the eval's id with --eval", eval) +} + +// EvalNotDeployedYet reports a declared eval that no deploy has created. +func EvalNotDeployedYet(eval, deployCmd string) error { + return fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `%s` first", eval, deployCmd) +} + +// NoEvalNamedOrDeclared reports a command with no eval to act on. +func NoEvalNamedOrDeclared(configPath string) error { + return fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + filepath.ToSlash(configPath)) +} + +// ListingRuns reports a failure to list an eval's runs. +func ListingRuns(evalID string, err error) error { + return fmt.Errorf("listing runs for %q: %w", evalID, err) +} + +// EvalHasNoRunsLine reports an eval with no runs to list. +func EvalHasNoRunsLine(evalID string) string { + return fmt.Sprintf("Eval %s has no runs yet.\n", evalID) +} + +// EvalHasNoRuns reports an eval with no run to fall back on. +func EvalHasNoRuns(evalID string) error { + return fmt.Errorf("eval %s has no runs yet", evalID) +} + +// ReadingRun reports a failure to read the run the caller named. +func ReadingRun(runID string, err error) error { + return fmt.Errorf("reading run %s: %w", runID, err) +} + +// RunHeading opens the detail view of one run. +func RunHeading(runID string) string { + return fmt.Sprintf("Run %s\n", runID) +} + +// RunNameLine reports the run's name in the detail view. +func RunNameLine(name string) string { + return fmt.Sprintf(" name : %s\n", name) +} + +// RunStatusDetail reports the run's status in the detail view. +func RunStatusDetail(status string) string { + return fmt.Sprintf(" status : %s\n", status) +} + +// RunResultsLine reports the run's counts in the detail view. +func RunResultsLine(counts string) string { + return fmt.Sprintf(" results : %s\n", counts) +} + +// RunReportLine reports the run's one link in the detail view. +func RunReportLine(url string) string { + return fmt.Sprintf(" report : %s\n", url) +} + +// CountsSummary renders a run's verdict counts on one line. +func CountsSummary(passed, failed, errored int) string { + return fmt.Sprintf("%d passed, %d failed, %d errored", passed, failed, errored) +} + +// RunAlreadyFinished reports a cancel asked of a run that already ended. +func RunAlreadyFinished(runID, status string) error { + return fmt.Errorf("run %s already finished with status %q", runID, status) +} + +// CancellingRun reports the service refusing to cancel the run. +func CancellingRun(runID string, err error) error { + return fmt.Errorf("cancelling run %s: %w", runID, err) +} + +// RunIsNow reports the state a cancelled run moved to. +func RunIsNow(runID, status string) string { + return fmt.Sprintf("Run %s is now %s\n", runID, status) +} + +// RunNotFound reports a run id the eval does not hold. +func RunNotFound(runID, evalID string) error { + return fmt.Errorf("no run %q on eval %q", runID, evalID) +} + +// DeletingRun reports the service refusing to delete the run. +func DeletingRun(runID string, err error) error { + return fmt.Errorf("deleting run %s: %w", runID, err) +} + +// RunDeleted confirms a deleted run. +func RunDeleted(runID string) string { + return fmt.Sprintf("Deleted run %s\n", runID) +} + +// ReadingRunResults reports a failure to read a run's per-sample rows. +func ReadingRunResults(runID string, err error) error { + return fmt.Errorf("reading the results of run %s: %w", runID, err) +} + +// OutputItemNotFound reports an output item the run does not hold. +func OutputItemNotFound(itemID, runID string) error { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, runID) +} + +// ReadingOutputItem reports a failure to read one evaluated row. +func ReadingOutputItem(itemID string, err error) error { + return fmt.Errorf("reading output item %q: %w", itemID, err) +} + +// RunStatusHeading opens the per-sample view of a run. +func RunStatusHeading(runID, status string) string { + return fmt.Sprintf("Run %s status: %s\n", runID, status) +} + +// ResultTotals reports a run's verdict counts above the rows. +func ResultTotals(passed, failed, errored int) string { + return fmt.Sprintf("Totals: %d passed, %d failed, %d errored\n\n", passed, failed, errored) +} + +// NoFailingRows reports a --failed-only listing with nothing in it. +func NoFailingRows() string { + return "\nNo failing rows.\n" +} + +// NoRowsScored reports a run that has produced no rows yet. +func NoRowsScored() string { + return "\nNo rows have been scored yet.\n" +} + +// SamplesNeedingALook closes a --failed-only listing, holding the rows that +// failed apart from the rows nothing managed to score. +// +// One count covering both contradicted the totals printed two lines above it, +// which is what a reader compares it with: a run reporting 5 failed and 8 +// errored closed with "13 sample(s) failed at least one evaluator". +func SamplesNeedingALook(failed, unscored int) string { + if unscored == 0 { + return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", failed) + } + if failed == 0 { + return fmt.Sprintf("\n%d sample(s) could not be scored.\n", unscored) + } + return fmt.Sprintf( + "\n%d sample(s) failed at least one evaluator, and %d could not be scored.\n", + failed, unscored) +} + +// GateSawUnscoredRows warns that a pass-rate gate judged only part of the run. +// +// The rate excludes rows nothing could grade, so a run that errored on most of +// its samples can clear a threshold on the few that survived. The gate is the +// one place a pipeline is guaranteed to read, so it is said there rather than +// left for someone to notice in the summary. +func GateSawUnscoredRows(unscored, total int) error { + return fmt.Errorf( + "%d of %d samples were not scored, so the pass rate this gate read covers "+ + "only the rest; use --fail-on any-failure to count them against the run", + unscored, total) +} + +// GeneratedNameNotAFileName reports a generated artifact name that would not +// stay inside the output directory, or that would produce a file whose name is +// read as a flag by whatever the path is handed to next. +func GeneratedNameNotAFileName(kind, name string) error { + return fmt.Errorf( + "%s name %q cannot be used as a file name: remove any of / \\ : , "+ + "do not start with -, and do not use . or ..", + kind, name) +} + +// OutputItemEmpty reports a row the service acknowledged but returned nothing +// for, which is a service fault rather than a missing item. +func OutputItemEmpty() error { + return errors.New("the service returned no content for this output item") +} + +// NotARegularFile reports an --output-file that names a directory or a device. +func NotARegularFile(path string) error { + return fmt.Errorf("%s is not a regular file, so it will not be overwritten", filepath.ToSlash(path)) +} + +// CannotWriteInDirectory reports a destination directory that cannot be written +// to. A missing directory is reported as such: the wrapped error names the +// temporary file the writer chose, which the caller never asked for. +func CannotWriteInDirectory(dir string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%s does not exist", filepath.ToSlash(dir)) + } + return fmt.Errorf("cannot write in %s: %w", dir, err) +} + +// OutputItemVerdict is one evaluator's line in `run output show`. +func OutputItemVerdict(evaluator, score, verdict string) string { + return fmt.Sprintf("%s %s %s\n", evaluator, score, verdict) +} + +// OutputItemEvaluator heads the dimensions of a rubric that scored per metric. +func OutputItemEvaluator(evaluator string) string { + return evaluator + "\n" +} + +// OutputItemMetric is one scored dimension under its evaluator. +func OutputItemMetric(metric, score, verdict string) string { + return fmt.Sprintf(" %s %s %s\n", metric, score, verdict) +} + +// OutputItemReason is the judge's explanation, indented under its verdict. +func OutputItemReason(reason string) string { + return fmt.Sprintf(" %s\n", reason) +} + +// ReportLinkAfterRows closes a per-sample listing with the run's one link. +func ReportLinkAfterRows(url string) string { + return fmt.Sprintf("\nReport: %s\n", url) +} + +// ExportFormatUnsupported reports an --format the export command cannot write. +func ExportFormatUnsupported(format, csv, json, jsonl string) error { + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, csv, json, jsonl) +} + +// FailOnInvalid reports a --fail-on value that is neither form of threshold. +func FailOnInvalid(spec string) error { + return fmt.Errorf("--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) +} + +// FailOnRateNotNumber reports a --fail-on pass rate that will not parse. +func FailOnRateNotNumber(rate string) error { + return fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) +} + +// FailOnRateOutOfRange reports a --fail-on pass rate outside 0..1. +func FailOnRateOutOfRange(value float64) error { + return fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) +} + +// GateNoResultCounts reports a gate that has nothing to measure against. +func GateNoResultCounts() string { + return "the run reported no result counts, so the threshold cannot be checked" +} + +// GateSamplesDidNotPass reports an any-failure gate that was breached. +func GateSamplesDidNotPass(unpassed, total int) string { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, total) +} + +// GateNoRowsScored reports a pass-rate gate over a run that scored nothing. +func GateNoRowsScored() string { + return "the run scored no rows, so its pass rate is below any threshold" +} + +// GatePassRateBelow reports a pass-rate gate that was breached. +// +// One decimal, which is what the spec's hero scenario shows -- except when that +// rounds the actual rate onto the threshold. The gate compares exact values, so +// 7996/10000 breaches 0.8 while both read "80.0%", and the line would say a +// rate is below itself. Only that case is given more precision. +func GatePassRateBelow(actual, required float64) string { + shown := fmt.Sprintf("%.1f", actual*100) + if shown == fmt.Sprintf("%.1f", required*100) { + shown = strconv.FormatFloat(actual*100, 'f', -1, 64) + } + return fmt.Sprintf("pass rate %s%% is below the required %.1f%%", + shown, required*100) +} + +// GateBreached is the block a breached gate leaves in a pipeline's log. +func GateBreached(reason string) string { + return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", + failedMark, reason) +} + +// --------------------------------------------------------------------------- +// Generation +// --------------------------------------------------------------------------- + +// GeneratedNameNeedsATarget reports a generation that can name neither the +// artifact nor the agent to derive its name from. +func GeneratedNameNeedsATarget(kind string) error { + return fmt.Errorf( + "no name for the generated %s and no target to derive one from: "+ + "pass --%s-name, or --target", kind, kind) +} + +// GenerationFailed labels one half of a composite generate that did not finish. +// +// The label goes inside a structured error rather than around it: azd +// serializes a LocalError's own message and drops any wrapper, so wrapping +// would throw away the one word saying which job failed. +func GenerationFailed(kind string, err error) error { + var local *azdext.LocalError + if errors.As(err, &local) { + labelled := *local + labelled.Message = "generating the " + kind + ": " + local.Message + return &labelled + } + return fmt.Errorf("generating the %s: %w", kind, err) +} + +// multiError presents several failures as one line while keeping every cause +// reachable through errors.Is and errors.As. +// +// errors.Join would keep the causes but renders them one per line, and this is +// a single error the CLI prints after "ERROR: ". +type multiError struct { + msg string + causes []error +} + +func (m *multiError) Error() string { return m.msg } +func (m *multiError) Unwrap() []error { return m.causes } + +// SomeGenerationsFailed reports a composite generate where at least one job +// did not finish. The others may well have. +// +// Two structured failures of the same category stay structured, so an expired +// login still arrives as an auth error carrying its suggestion rather than as +// a flat string. +func SomeGenerationsFailed(failures []error) error { + if len(failures) == 1 { + return failures[0] + } + + parts := make([]string, 0, len(failures)) + for _, f := range failures { + parts = append(parts, f.Error()) + } + joined := strings.Join(parts, "; ") + + var first *azdext.LocalError + if !errors.As(failures[0], &first) { + return &multiError{msg: joined, causes: failures} + } + for _, f := range failures[1:] { + var other *azdext.LocalError + if !errors.As(f, &other) || other.Category != first.Category { + return &multiError{msg: joined, causes: failures} + } + } + merged := *first + merged.Message = joined + return &merged +} + +// GenerationStarting announces a job before it is submitted, so a long +// generation is not silent while it runs. +func GenerationStarting(kind, name string) string { + return fmt.Sprintf(" Starting %s generation for %q...\n", kind, name) +} + +// GenerationModelRequired reports a generation with no deployment to run on. +// +// Reached only when the target agent could not supply one either, so the flag +// is the whole of the way out. +func GenerationModelRequired() error { + return errors.New("a model deployment is required to generate: pass --generation-model") +} + +// ReadingInstructionFile reports an --agent-instruction-file that would not read. +func ReadingInstructionFile(path string, err error) error { + return fmt.Errorf("reading --agent-instruction-file %q: %w", filepath.ToSlash(path), err) +} + +// InstructionFileEmpty reports an --agent-instruction-file with nothing in it. +func InstructionFileEmpty(path string) error { + return fmt.Errorf("--agent-instruction-file %q is empty", filepath.ToSlash(path)) +} + +// ReadingInstructions reports a declared instructions file that would not read. +func ReadingInstructions(named string, err error) error { + return fmt.Errorf("reading instructions %q: %w", named, err) +} + +// SeedingFromFile names the local file generation was seeded from. +func SeedingFromFile(path string) string { + return fmt.Sprintf(" Seeding generation from %s.\n", filepath.ToSlash(path)) +} + +// SeedingFromAgent names the agent whose published instructions seeded generation. +func SeedingFromAgent(agent string) string { + return fmt.Sprintf(" Seeding generation from the instructions of agent %q.\n", agent) +} + +// WarningAgentUnreadable reports an agent that could not supply context. +// +// A misspelled --target is the common cause and answers 404, whose body is ten +// lines of URL, status rule and nested JSON for a fact that fits on one. +func WarningAgentUnreadable(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so generation "+ + "has no agent context to work from\n", agent) + } + return fmt.Sprintf(" warning: could not read agent %q for generation context: %v\n", + agent, err) +} + +// notFound reports a service answer of 404. +// +// Written here rather than imported from eval_api, because that package imports +// this one for its own wording and the dependency only goes one way. +func notFound(err error) bool { + var respErr *azcore.ResponseError + return errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound +} + +// WarningAgentSeedFailedRetrying reports the retry that drops the agent source. +func WarningAgentSeedFailedRetrying(agent string) string { + return fmt.Sprintf( + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", agent) +} + +// GeneratingRubric reports a rubric generation job about to be submitted. +func GeneratingRubric(name string) string { + return fmt.Sprintf("Generating rubric %s...\n", name) +} + +// GeneratingDataset reports a dataset generation job about to be submitted. +func GeneratingDataset(name string, samples int) string { + return fmt.Sprintf("Generating dataset %s (%d samples)...\n", name, samples) +} + +// SubmittingRubricJob reports the service refusing the rubric job. +func SubmittingRubricJob(err error) error { + return fmt.Errorf("submitting the rubric generation job: %w", err) +} + +// SubmittingDataJob reports the service refusing the data generation job. +func SubmittingDataJob(err error) error { + return fmt.Errorf("submitting the data generation job: %w", err) +} + +// RubricGeneration reports a rubric job that did not finish successfully. +func RubricGeneration(err error) error { + return fmt.Errorf("rubric generation: %w", err) +} + +// DataGeneration reports a data job that did not finish successfully. +func DataGeneration(err error) error { + return fmt.Errorf("data generation: %w", err) +} + +// RubricJobReturnedNoResult reports a completed rubric job with nothing to write. +func RubricJobReturnedNoResult() error { + return errors.New("the rubric generation job returned no result") +} + +// DataJobReturnedNoDataset reports a completed data job with nothing to fetch. +func DataJobReturnedNoDataset() error { + return errors.New("the data generation job returned no dataset reference") +} + +// ReadingGeneratedDataset reports the generated dataset not being there to read. +func ReadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("reading the generated dataset %q: %w", name, err) +} + +// DownloadingGeneratedDataset reports a failure to fetch the generated rows. +func DownloadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("downloading the generated dataset %q: %w", name, err) +} + +// AgentSeededGenerationFailing explains the service-side failure that hits +// every agent, so the caller does not retry against a deterministic failure. +func AgentSeededGenerationFailing(err error, agent string) error { + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agent) +} + +// FromPromptNeedsInstruction reports --from prompt with nothing to prompt with. +func FromPromptNeedsInstruction() string { + return "--from prompt needs --agent-instruction or --agent-instruction-file" +} + +// FromAgentNeedsTarget reports --from agent with no agent to read. +func FromAgentNeedsTarget() string { + return "--from agent needs a target agent; pass --target, " + + "or declare one under target: in azure.eval.yaml" +} + +// FromFileNotASource reports --from file, which generation has no path for. +func FromFileNotASource() string { + return "--from file is not a generation source; " + + "register the file with `azd ai eval dataset create` instead" +} + +// FromNotBuildable reports a --from this plan cannot satisfy. +func FromNotBuildable(kind string) string { + return fmt.Sprintf("--from %s cannot be built from this plan", kind) +} + +// UnbuildableSources reports every --from the plan could not honour at once. +func UnbuildableSources(reasons []string) error { + return errors.New(strings.Join(reasons, "; ")) +} + +// JobSubmitted reports the id of a job started with --no-wait. +func JobSubmitted(jobID string) string { + return fmt.Sprintf(" submitted job %s\n", jobID) +} + +// ReattachToJob says how to come back to a job started with --no-wait. +// +// The selector is part of the line because `job` requires it: the two +// collections share an id shape, so an id alone does not say which to call. +func ReattachToJob(selector, jobID string) string { + return fmt.Sprintf("\nReattach with: azd ai eval job show %s --%s\n", jobID, selector) +} + +// WroteArtifact reports where a generated artifact landed. +func WroteArtifact(path string) string { + return fmt.Sprintf("%s Downloaded %s\n", doneMark, filepath.ToSlash(path)) +} + +// ArtifactExists reports a generation that would overwrite a checked-in file. +func ArtifactExists(path string) error { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + path) +} + +// JobKindRequired reports a job command that does not say which collection. +func JobKindRequired() error { + return errors.New("pass --dataset or --evaluator to say which generation jobs to act on") +} + +// ListingJobs reports a failure to list one kind of generation job. +func ListingJobs(kind string, err error) error { + return fmt.Errorf("listing %s generation jobs: %w", kind, err) +} + +// NoJobs reports a project with no generation jobs of that kind. +func NoJobs(kind string) string { + return fmt.Sprintf("No %s generation jobs found.\n", kind) +} + +// JobLine renders one generation job in a listing or a detail view. +func JobLine(jobID, status string) string { + return fmt.Sprintf("%s %s\n", jobID, status) +} + +// JobErrorLine reports why a generation job failed. +func JobErrorLine(message string) string { + return fmt.Sprintf("error: %s\n", message) +} + +// JobCancelled confirms a cancelled generation job. +func JobCancelled(kind, jobID, status string) string { + return fmt.Sprintf("Cancelled %s generation job %s (%s)\n", kind, jobID, status) +} + +// JobDeleted confirms a deleted generation job record. +func JobDeleted(kind, jobID string) string { + return fmt.Sprintf("Deleted %s generation job %s\n", kind, jobID) +} + +// JobNotFound reports a job id that is not in this group, naming the other one. +// +// Phrased to avoid an article before the kind: "a evaluator" is what the +// obvious wording produces. +func JobNotFound(kind, jobID, other string) error { + return fmt.Errorf( + "no %s generation job %q in this project; try the %s job group", + kind, jobID, other) +} + +// JobActionFailed reports a job operation that was not a read, so the sentence +// names what was attempted. A delete that reports "reading" sends the reader +// looking for a read that never happened. +func JobActionFailed(action, kind, jobID string, err error) error { + return fmt.Errorf("%s %s generation job %s: %w", action, kind, jobID, err) +} + +// JobFailedWithReason reports a polled job that failed and said why. +func JobFailedWithReason(status, message string) string { + return fmt.Sprintf("job failed with status %q: %s", status, message) +} + +// JobFailed reports a polled job that failed without saying why. +func JobFailed(status string) string { + return fmt.Sprintf("job failed with status %q", status) +} + +// PollerTimedOut reports a job that was still running when polling gave up. +func PollerTimedOut(operationID string, attempts int) string { + return fmt.Sprintf("operation %s did not complete within %d attempts", + operationID, attempts) +} + +// OperationIDEmpty reports a poll with nothing to poll for. +func OperationIDEmpty() error { + return errors.New("operation ID is empty") +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// ReadingDataset reports a dataset that could not be read, by name or by path. +func ReadingDataset(dataset string, err error) error { + return fmt.Errorf("reading dataset %q: %w", dataset, err) +} + +// DatasetHasNoVersionsToRead reports a registered dataset with nothing published. +func DatasetHasNoVersionsToRead(dataset string) error { + return fmt.Errorf("dataset %q has no versions to read", dataset) +} + +// ReadingDatasetVersion reports one version of a dataset failing to read. +func ReadingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("reading dataset %q version %s: %w", dataset, version, err) +} + +// CheckingDataset reports the read that decides whether a name is already +// taken. It is worth its own message because that read is what separates +// `create` from `update`, and a failure answered as "not there" turns a create +// into a silent update. +func CheckingDataset(dataset string, err error) error { + return fmt.Errorf( + "checking whether dataset %q already exists: %w", dataset, err) +} + +// DatasetVersionEmpty reports a published version that holds no rows. +func DatasetVersionEmpty(dataset, version string) error { + return fmt.Errorf("dataset %q version %s has no rows", dataset, version) +} + +// JSONLLineInvalid reports a row that is not JSON, by line. +func JSONLLineInvalid(line int, err error) error { + return fmt.Errorf("line %d is not valid JSON: %w", line, err) +} + +// JSONLRowInvalid reports a row that is not JSON before the file is published. +func JSONLRowInvalid(path string, line int, err error) error { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) +} + +// JSONLRowEmpty reports a row that parses to nothing to evaluate. +func JSONLRowEmpty(path string, line int) error { + return fmt.Errorf("%s line %d is an empty object, which evaluates to nothing", path, line) +} + +// JSONLNoRows reports a dataset file with nothing in it to evaluate. +func JSONLNoRows(path string) error { + return fmt.Errorf("%s has no rows to evaluate", filepath.ToSlash(path)) +} + +// ReadingFromFile reports a --from-file that would not stat. +func ReadingFromFile(path string, err error) error { + return fmt.Errorf("reading --from-file %q: %w", filepath.ToSlash(path), err) +} + +// FromFileMustBeJSONL reports a --from-file that is not a dataset. +func FromFileMustBeJSONL(path string) error { + return fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", + filepath.ToSlash(path)) +} + +// FromFileDirectoryHasNoJSONL reports a directory with nothing to upload. +func FromFileDirectoryHasNoJSONL(dir string) error { + return fmt.Errorf("no .jsonl file in %q; --from-file needs one to upload", filepath.ToSlash(dir)) +} + +// FromFileDirectoryIsAmbiguous refuses to guess which dataset was meant. +func FromFileDirectoryIsAmbiguous(dir string, names []string) error { + return fmt.Errorf( + "%q holds %d .jsonl files (%s); name the one to upload with --from-file", + filepath.ToSlash(dir), len(names), strings.Join(names, ", ")) +} + +// InvalidDatasetName reports a name the service will not accept. +func InvalidDatasetName(name string) error { + return invalidAssetName("dataset", name) +} + +// InvalidEvaluatorName reports a name the service will not accept. +func InvalidEvaluatorName(name string) error { + return invalidAssetName("evaluator", name) +} + +func invalidAssetName(kind, name string) error { + return fmt.Errorf( + "%s name %q is invalid: use letters, digits, dashes and underscores, "+ + "up to 255 characters", kind, name) +} + +// RegisteringDataset reports the service refusing to publish the dataset. +func RegisteringDataset(dataset string, err error) error { + return fmt.Errorf("registering dataset %q: %w", dataset, err) +} + +// DatasetRegistered confirms a published dataset version. +func DatasetRegistered(dataset, version string) string { + return fmt.Sprintf("Registered dataset %s version %s\n", dataset, version) +} + +// ListingDatasets reports a failure to list the project's datasets. +func ListingDatasets(err error) error { + return fmt.Errorf("listing datasets: %w", err) +} + +// ListingDatasetVersions reports a failure to list one dataset's versions. +func ListingDatasetVersions(dataset string, err error) error { + return fmt.Errorf("listing versions of dataset %q: %w", dataset, err) +} + +// NoDatasets reports a project with no datasets to list. +func NoDatasets() string { + return "No datasets found.\n" +} + +// NoDatasetVersions reports a name whose versions listed nothing. +// +// Listing a name that does not exist is not an error — a delete is checked for +// idempotence this way — so this has to read as an answer about that name +// rather than as a report about the project, which holds other datasets. +// +// The suggested command carries no placeholder, so it pastes and runs; the file +// is the one thing only the caller knows, and is named outside the command. +func NoDatasetVersions(dataset string) string { + return fmt.Sprintf("No versions of dataset %q. Publish one with "+ + "`azd ai eval dataset create %s` and a --from-file path.\n", dataset, dataset) +} + +// ResolvingLatestDatasetVersion reports a failure to find what "latest" means. +func ResolvingLatestDatasetVersion(dataset string, err error) error { + return fmt.Errorf("resolving the latest version of %q: %w", dataset, err) +} + +// DatasetNotFound reports a name the project does not hold. +// +// The service answers an unknown name with an empty version list rather than a +// 404, and a dataset cannot exist with no versions, so an empty list means the +// dataset is absent rather than empty. +func DatasetNotFound(dataset string) error { + return fmt.Errorf( + "no dataset %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", dataset) +} + +// DatasetVersionNotFoundWithHint reports a dataset version the project does not hold. +func DatasetVersionNotFoundWithHint(dataset, version string) error { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset versions list %s` shows the ones there are", dataset, version, dataset) +} + +// DatasetVersionNotFound reports a dataset version there is nothing to delete at. +func DatasetVersionNotFound(dataset, version string) error { + return fmt.Errorf("no dataset %q at version %q in this project", dataset, version) +} + +// DeletingDatasetVersion reports the service refusing the delete. +func DeletingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("deleting dataset %q version %q: %w", dataset, version, err) +} + +// DatasetDeleted confirms a deleted dataset version. +func DatasetDeleted(dataset, version string) string { + return fmt.Sprintf("Deleted dataset %s version %s\n", dataset, version) +} + +// DatasetProblem attributes a failure to the dataset it happened under. +func DatasetProblem(dataset string, err error) error { + return fmt.Errorf("dataset %q: %w", dataset, err) +} + +// DatasetSource reports a declared source that is not on disk. +func DatasetSource(path string, err error) error { + return fmt.Errorf("dataset source %q: %w", filepath.ToSlash(path), err) +} + +// DatasetNotGeneratedYet reports a declared dataset whose rows are not written +// yet. +// +// `init` declares the dataset it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with DatasetProblem, which names the dataset, so this +// does not name it again. +func DatasetNotGeneratedYet(dataset, path string) error { + return fmt.Errorf( + "its rows %s have not been generated yet. "+ + "Run `azd ai eval generate --dataset --dataset-name %s` to write them, "+ + "or point the declaration at a .jsonl you already have", + filepath.ToSlash(path), dataset) +} + +// DatasetNotLocalNorFound reports a source-less dataset the project rejected. +func DatasetNotLocalNorFound(dataset string, err error) error { + return fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + dataset, err) +} + +// DatasetNotLocalNorRegistered reports a source-less dataset nobody published. +func DatasetNotLocalNorRegistered(dataset string) error { + return fmt.Errorf( + "dataset %q has no local source and is not registered on the project", dataset) +} + +// DatasetVersionConflict reports a pinned version the local file disagrees with. +func DatasetVersionConflict(dataset, version string) error { + return fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + dataset, version) +} + +// DatasetDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval dataset update` publishes without recording the per-dataset +// version the reconciler reads, so it is a likely cause and naming it saves the +// reader looking for a colleague who did nothing. "Pull the newer content +// locally" was the other half of the old advice and is a no-op when the bytes +// already match, which is the common case. +func DatasetDrifted(dataset, latest, recorded string) error { + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "something published outside this configuration, which `azd ai eval dataset update` "+ + "on the same dataset also does. Pin it with `version: %s` on the dataset to deploy "+ + "what is already there, or publish a new version from the configuration's source, "+ + "then deploy again", + dataset, latest, recorded, latest) +} + +// ReadingDatasetDirectory reports the upload scan failing to read the directory. +func ReadingDatasetDirectory(err error) error { + return fmt.Errorf("reading directory: %w", err) +} + +// DatasetFileHasNoRows reports an empty dataset file, refused before upload. +func DatasetFileHasNoRows(name string) error { + return fmt.Errorf( + "dataset file %q has no rows, so there would be nothing to evaluate", name) +} + +// NoJSONLInDirectory reports an upload directory holding no dataset. +func NoJSONLInDirectory(dir string) error { + return fmt.Errorf("no .jsonl file found in %s", filepath.ToSlash(dir)) +} + +// ReadingDatasetFromDir reports the upload failing to gather the local rows. +func ReadingDatasetFromDir(dir string, err error) error { + return fmt.Errorf("reading dataset from %s: %w", dir, err) +} + +// StartingPendingUpload reports the service refusing to open an upload. +func StartingPendingUpload(err error) error { + return fmt.Errorf("starting pending upload: %w", err) +} + +// NoUploadURI reports an accepted upload the service gave nowhere to write to. +func NoUploadURI() error { + return errors.New("no upload SAS URI returned from startPendingUpload") +} + +// UploadingBlob reports the dataset content failing to upload. +func UploadingBlob(err error) error { + return fmt.Errorf("uploading blob: %w", err) +} + +// ReadingDownloadCredentials reports the service refusing to hand out a read URI. +func ReadingDownloadCredentials(dataset string, err error) error { + return fmt.Errorf("reading download credentials for %q: %w", dataset, err) +} + +// NoDownloadURI reports a dataset the service gave nowhere to read from. +func NoDownloadURI(dataset string) error { + return fmt.Errorf("no download URI returned for dataset %q", dataset) +} + +// ListingDatasetContent reports a failure to list what a dataset version holds. +func ListingDatasetContent(dataset string, err error) error { + return fmt.Errorf("listing the content of dataset %q: %w", dataset, err) +} + +// DatasetHasNoFile reports a dataset version with nothing to download. +func DatasetHasNoFile(dataset string) error { + return fmt.Errorf("dataset %q holds no downloadable file", dataset) +} + +// --------------------------------------------------------------------------- +// Evaluators +// --------------------------------------------------------------------------- + +// EvaluatorNeedsFields reports required inputs the dataset does not carry. +func EvaluatorNeedsFields(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or bind it with `data_mapping`", + evaluator, quoteList(missing), pluralColumns(missing)) +} + +// EvaluatorLevelUnsupported reports an evaluation level the evaluator refuses. +func EvaluatorLevelUnsupported(evaluator, level string, supported []string) error { + return fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + evaluator, level, quoteList(supported)) +} + +// EvaluatorNeedsInitParams reports required initialization parameters left unset. +func EvaluatorNeedsInitParams(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", + evaluator, quoteList(missing)) +} + +// ReadingEvaluator reports an evaluator that could not be read, by name or path. +func ReadingEvaluator(evaluator string, err error) error { + return fmt.Errorf("reading evaluator %q: %w", evaluator, err) +} + +// EvaluatorProblem attributes a failure to the evaluator it happened under. +func EvaluatorProblem(evaluator string, err error) error { + return fmt.Errorf("evaluator %q: %w", evaluator, err) +} + +// EvaluatorSource reports a declared source that is not on disk. +func EvaluatorSource(path string, err error) error { + return fmt.Errorf("evaluator source %q: %w", filepath.ToSlash(path), err) +} + +// EvaluatorNotGeneratedYet reports a declared evaluator whose definition has +// not been written yet. +// +// `init` declares the rubric it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with EvaluatorProblem, which names the evaluator, so +// this does not name it again. +func EvaluatorNotGeneratedYet(evaluator, path string) error { + return fmt.Errorf( + "its definition %s has not been generated yet. "+ + "Run `azd ai eval generate --evaluator --evaluator-name %s` to write it, "+ + "or drop the evaluator from azure.eval.yaml", + filepath.ToSlash(path), evaluator) +} + +// CheckingEvaluatorExists reports a failure to tell create from update. +func CheckingEvaluatorExists(evaluator string, err error) error { + return fmt.Errorf("checking whether evaluator %q exists: %w", evaluator, err) +} + +// RegisteringEvaluator reports the service refusing to publish the evaluator. +func RegisteringEvaluator(evaluator string, err error) error { + return fmt.Errorf("registering evaluator %q: %w", evaluator, err) +} + +// EvaluatorRegistered confirms a published evaluator version. +func EvaluatorRegistered(evaluator, version string) string { + return fmt.Sprintf("Registered evaluator %s version %s\n", evaluator, version) +} + +// AssetAlreadyExists reports `create` asked of a name already in use. +func AssetAlreadyExists(kind, name string) error { + return fmt.Errorf("%s %q already exists: use `update` to publish a new version", kind, name) +} + +// AssetDoesNotExist reports `update` asked of a name nobody registered. +func AssetDoesNotExist(kind, name string) error { + return fmt.Errorf("%s %q does not exist: use `create` to register it", kind, name) +} + +// DefinitionNotJSONObject reports an evaluator definition that is not an object. +func DefinitionNotJSONObject(err error) error { + return fmt.Errorf("the definition is not a JSON object: %w", err) +} + +// NotValidJSON reports an evaluator file that will not parse at all. +func NotValidJSON(err error) error { + return fmt.Errorf("not valid JSON: %w", err) +} + +// RubricMissingDimensions reports a file that is neither rubric nor document. +func RubricMissingDimensions() error { + return errors.New( + "expected a rubric definition with 'dimensions', or a document with 'definition'") +} + +// ListingEvaluators reports a failure to list the project's evaluators. +func ListingEvaluators(err error) error { + return fmt.Errorf("listing evaluators: %w", err) +} + +// ListingEvaluatorVersions reports a failure to list one evaluator's versions. +func ListingEvaluatorVersions(evaluator string, err error) error { + return fmt.Errorf("listing versions of evaluator %q: %w", evaluator, err) +} + +// NoEvaluators reports a project with no evaluators to list. +func NoEvaluators() string { + return "No evaluators found.\n" +} + +// EvaluatorNotFound reports an evaluator the project does not hold. +func EvaluatorNotFound(evaluator string) error { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", evaluator) +} + +// EvaluatorVersionNotFound reports an evaluator version there is nothing to delete at. +func EvaluatorVersionNotFound(evaluator, version string) error { + return fmt.Errorf("no evaluator %q at version %q in this project", evaluator, version) +} + +// DeletingEvaluatorVersion reports the service refusing the delete. +func DeletingEvaluatorVersion(evaluator, version string, err error) error { + return fmt.Errorf("deleting evaluator %q version %q: %w", evaluator, version, err) +} + +// EvaluatorDeleted confirms a deleted evaluator version. +func EvaluatorDeleted(evaluator, version string) string { + return fmt.Sprintf("Deleted evaluator %s version %s\n", evaluator, version) +} + +// EvaluatorNotLocalNorFound reports a source-less evaluator the project rejected. +func EvaluatorNotLocalNorFound(evaluator string, err error) error { + return fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + evaluator, err) +} + +// EvaluatorDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval evaluator update` publishes without recording the version the +// reconciler reads, so it is a likely cause and naming it saves the reader +// looking for a colleague who did nothing. +func EvaluatorDrifted(evaluator, remote, recorded string) error { + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: something published a "+ + "version outside this configuration, which `azd ai eval evaluator update` on "+ + "the same evaluator also does. Publishing over it would leave that change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + evaluator, remote, recorded, remote) +} + +// EvaluatorVersionNotAdvancing reports a publish the service kept answering with +// a version that already existed. +func EvaluatorVersionNotAdvancing(evaluator, version string, waited fmt.Stringer) error { + return fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + evaluator, version, waited, version) +} + +// EvaluatorHasNoVersions reports an evaluator nothing was ever published under. +func EvaluatorHasNoVersions(evaluator string) error { + return fmt.Errorf("evaluator %q has no versions", evaluator) +} + +// EvaluatorHasNoUsableVersion reports versions none of which can be resolved. +func EvaluatorHasNoUsableVersion(evaluator string) error { + return fmt.Errorf("evaluator %q has no usable version", evaluator) +} + +// BareEvaluatorEntry reports an evaluators: entry written as a plain string. +func BareEvaluatorEntry(name string) error { + return fmt.Errorf( + "an evaluator entry is a mapping, not a bare string: "+ + "write `- evaluator: %s`", name) +} + +// EvaluatorsMustBeSequence reports an evaluators: block that is not a list. +func EvaluatorsMustBeSequence(kind any) error { + return fmt.Errorf("evaluators must be a sequence, got %v", kind) +} + +// EvaluatorsMustBeList reports an evaluators: block that is not a JSON array. +func EvaluatorsMustBeList(err error) error { + return fmt.Errorf("evaluators must be a list: %w", err) +} + +// DecodingEvaluatorName reports an evaluator entry whose name will not decode. +func DecodingEvaluatorName(err error) error { + return fmt.Errorf("decoding evaluator name: %w", err) +} + +// DecodingEvaluator reports an evaluator entry that will not decode. +func DecodingEvaluator(err error) error { + return fmt.Errorf("decoding evaluator: %w", err) +} + +// EvaluatorEntryMissingEvaluator reports an entry that names no evaluator. +func EvaluatorEntryMissingEvaluator() error { + return errors.New("evaluator entry is missing 'evaluator'") +} + +// EvaluatorEntryMustBeMapping reports an entry that is neither map nor string. +func EvaluatorEntryMustBeMapping(kind any) error { + return fmt.Errorf("evaluator entry must be a mapping, got %v", kind) +} + +// --------------------------------------------------------------------------- +// Deploy and reconcile +// --------------------------------------------------------------------------- + +// EvalConfigInvalid reports a configuration a deploy will not act on. +func EvalConfigInvalid(err error) error { + return fmt.Errorf("eval config is invalid: %w", err) +} + +// ServiceCarriesNoConfig reports an azure.yaml entry with nothing to deploy. +func ServiceCarriesNoConfig(service string) error { + return fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evals", + service) +} + +// ResolvingServiceRefs reports a $ref that could not be followed. +func ResolvingServiceRefs(err error) error { + return fmt.Errorf("resolving $ref in the eval service configuration: %w", err) +} + +// ReadingServiceConfig reports the service entry failing to serialize. +func ReadingServiceConfig(err error) error { + return fmt.Errorf("reading the eval service configuration: %w", err) +} + +// ReconcilingDataset reports the dataset a deploy has reached. +func ReconcilingDataset(dataset string) string { + return fmt.Sprintf("Reconciling dataset %s", dataset) +} + +// ReconcilingEvaluator reports the evaluator a deploy has reached. +func ReconcilingEvaluator(evaluator string) string { + return fmt.Sprintf("Reconciling evaluator %s", evaluator) +} + +// ReconcilingEval reports the eval a deploy has reached. +func ReconcilingEval(eval string) string { + return fmt.Sprintf("Reconciling eval %s", eval) +} + +// PublishedVersion reports an artifact a deploy published. +func PublishedVersion(kind, name, version string) string { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) +} + +// UnchangedAtVersion reports an artifact a deploy left alone. +func UnchangedAtVersion(kind, name, version string) string { + return fmt.Sprintf("%s %s is unchanged at version %s", + strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +// EvalProblem attributes a failure to the eval it happened under. +func EvalProblem(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) +} + +// EvalCreatedProgress and EvalUnchangedProgress are the deploy-time equivalents +// of EvalCreated and EvalUnchanged, without the status marks azd adds itself. +func EvalCreatedProgress(eval, id string) string { + return fmt.Sprintf("Created eval %s (%s)", eval, id) +} + +func EvalUnchangedProgress(eval, id string) string { + return fmt.Sprintf("Eval %s is unchanged (%s)", eval, id) +} + +// EvalCreated confirms a single eval created outside a full deploy. +func EvalCreated(eval, id string) string { + return fmt.Sprintf("%s Created eval: %s (%s)\n", doneMark, eval, id) +} + +// EvalUnchanged reports an eval a create found already in place. +// +// An eval is immutable, so re-running create against an unedited declaration +// creates nothing. Saying "Created" there claims work that did not happen, and +// hides the one thing worth checking: that the id, and so the run history +// hanging off it, survived. +func EvalUnchanged(eval, id string) string { + return fmt.Sprintf("%s Eval %s is unchanged (%s)\n", skippedMark, eval, id) +} + +// ListingEvals reports a failure to list the project's evals. +func ListingEvals(err error) error { + return fmt.Errorf("listing evals: %w", err) +} + +// NoEvals reports a project with no evals to list. +func NoEvals() string { + return "No evals found.\n" +} + +// EvalNotFound reports an eval id the project does not hold. +func EvalNotFound(evalID string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) +} + +// AmbiguousEvalName reports a name carried by more than one eval. +// +// An eval is immutable, so editing a declaration creates another under the same +// name and leaves the previous one holding its run history. Deleting takes the +// runs with it, so which one is meant has to be said rather than guessed. +func AmbiguousEvalName(name string, ids []string) error { + return fmt.Errorf( + "%d evals are named %q, and deleting one discards its runs, so name the "+ + "id instead: %s", len(ids), name, strings.Join(ids, ", ")) +} + +// EvalGone reports an eval id there is nothing to delete at. +func EvalGone(evalID string) error { + return fmt.Errorf("no eval %q in this project", evalID) +} + +// ReadingEval reports a failure to read one eval. +func ReadingEval(evalID string, err error) error { + return fmt.Errorf("reading eval %q: %w", evalID, err) +} + +// DeletingEval reports the service refusing the delete. +func DeletingEval(evalID string, err error) error { + return fmt.Errorf("deleting eval %q: %w", evalID, err) +} + +// EvalDeleted confirms a deleted eval. +func EvalDeleted(evalID string) string { + return fmt.Sprintf("Deleted eval %s\n", evalID) +} + +// Hashing reports a local artifact that could not be fingerprinted. +func Hashing(path string, err error) error { + return fmt.Errorf("hashing %q: %w", filepath.ToSlash(path), err) +} + +// HashingEval reports an eval declaration that could not be fingerprinted. +func HashingEval(eval string, err error) error { + return fmt.Errorf("hashing eval %q: %w", eval, err) +} + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +// NoAzdProject reports a command that found no project to attach to. +func NoAzdProject() error { + return errors.New( + "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml") +} + +// ConnectingToAzd reports the azd daemon being unreachable. +func ConnectingToAzd(err error) error { + return fmt.Errorf("connecting to azd: %w", err) +} + +// CreatingCredential reports the Azure credential failing to build. +func CreatingCredential(err error) error { + return fmt.Errorf("creating Azure credential: %w", err) +} + +// ErrNoAzdEnvironment reports that there is no azd environment to persist into. +var ErrNoAzdEnvironment = errors.New("no active azd environment") + +// NoAzdEnvironmentToWrite reports a value with nowhere to be remembered. +func NoAzdEnvironmentToWrite(key string) error { + return fmt.Errorf("%w to write %s into", ErrNoAzdEnvironment, key) +} + +// WritingEnvValue reports the azd environment refusing a write. +func WritingEnvValue(key string, err error) error { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) +} + +// BuildingServiceEntry reports the eval service entry failing to build. +func BuildingServiceEntry(err error) error { + return fmt.Errorf("building the eval service entry: %w", err) +} + +// AddingServiceTo reports azd refusing to add the eval service. +func AddingServiceTo(rootConfig string, err error) error { + return fmt.Errorf("adding the eval service to %s: %w", rootConfig, err) +} + +// SourceNotADataSource reports an --source that names nothing rows come from. +func SourceNotADataSource(source, dataset, traces string) error { + return fmt.Errorf("--source %q is not a data source; use %q or %q", source, dataset, traces) +} + +// TracesTakesNoDataset reports --dataset paired with a trace-backed eval. +func TracesTakesNoDataset() error { + return errors.New("--source traces reads production traces, so it takes no --dataset") +} + +// MaxTracesNeedsTraceSource reports --max-traces without a trace-backed eval. +func MaxTracesNeedsTraceSource() error { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") +} + +// MaxTracesMustBePositive reports a negative --max-traces. +func MaxTracesMustBePositive() error { + return errors.New("--max-traces must be positive") +} + +// EvalAlreadyDeclared reports an init that would overwrite a hand-tuned eval. +func EvalAlreadyDeclared(eval, configPath string) error { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + eval, configPath) +} + +// CreatingDatasetsDir reports the datasets directory failing to be created. +func CreatingDatasetsDir(err error) error { + return fmt.Errorf("creating the datasets directory: %w", err) +} + +// CreatingEvaluatorsDir reports the evaluators directory failing to be created. +func CreatingEvaluatorsDir(err error) error { + return fmt.Errorf("creating the evaluators directory: %w", err) +} + +// DetectedTarget reports the agent the scaffolded eval will evaluate. +func DetectedTarget(target string) string { + return fmt.Sprintf("%s Detected agent target: %s\n", doneMark, target) +} + +// NoAgentToEvaluate reports a project declaring no agent service. +func NoAgentToEvaluate() error { + return errors.New( + "this project declares no agent service to evaluate. Add one, or name an " + + "existing agent with --target") +} + +// AmbiguousAgentTarget reports several agents where only one can be scaffolded. +func AmbiguousAgentTarget(agents []string) error { + return fmt.Errorf( + "this project declares more than one agent (%s), so --target says which to "+ + "evaluate", strings.Join(agents, ", ")) +} + +// SelectAgentPrompt asks which agent the eval is for. +func SelectAgentPrompt() string { + return "Select the agent to evaluate:" +} + +// SelectingAgent reports a failed agent prompt. +func SelectingAgent(err error) error { + return fmt.Errorf("selecting an agent to evaluate: %w", err) +} + +// JudgeModelRequired reports a scaffold that has no deployment to judge with. +// +// Reached when the project declares no model deployment to read one from, so +// the flag is the whole of the way out. Failing here is deliberate: the judging +// built-ins declare the deployment as required, so a config written without one +// is rejected by the service later, far from the command that wrote it. +func JudgeModelRequired() error { + return errors.New( + "a model deployment is required to judge with: pass --judge-model. " + + "This project declares no deployments, and the azd environment sets no " + + "AZURE_AI_MODEL_DEPLOYMENT_NAME") +} + +// EvaluatorRefEmpty reports an --evaluator that carries no name, which is what +// a stray comma leaves behind. +func EvaluatorRefEmpty() error { + return errors.New("--evaluator was given an empty reference: name an evaluator, " + + "or use builtin. for a built-in") +} + +// EvaluatorRefMalformed reports a reference no evaluator can be found under. +func EvaluatorRefMalformed(ref string) error { + return fmt.Errorf("%q is not an evaluator reference: repeat --evaluator, or separate "+ + "them with commas, and use builtin. for a built-in", ref) +} + +// GateNeedsATerminalRun refuses to gate a run that is still moving. +// +// The counts are partial until the run stops, so a threshold read from them +// can fail a run that would have passed. Ignoring the flag instead would leave +// a pipeline believing it is gated when it is not. +func GateNeedsATerminalRun(runID, status string) error { + return fmt.Errorf( + "run %s is %s, so --fail-on has only partial results to judge: "+ + "add --wait to gate once it finishes", + runID, status) +} + +// InEvalAt says which declaration an error came from. +// +// The source rules are checked in one place and reported from two, because the +// same file is read when it is validated and again when a run is built. Only +// the first of those has an index to name. +func InEvalAt(i int, eval string, err error) error { + return fmt.Errorf("evals[%d] (%s): %w", i, eval, err) +} + +// InEval says which eval an error came from, where there is no index. +func InEval(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) +} + +// TraceWindowNotATime reports a window bound that is not a timestamp. +func TraceWindowNotATime(field, value string) error { + return fmt.Errorf( + "source.%s is %q, which is not a time: use RFC 3339, "+ + "for example 2026-08-18T09:00:00Z", + field, value) +} + +// TraceWindowBoundUnusable reports a bound that parses but says nothing. +// +// Both this layer and the wire read a zero as "no bound", so a bound that +// resolves to one would be dropped from the request rather than applied. +func TraceWindowBoundUnusable(field, value string) error { + return fmt.Errorf( + "source.%s is %q, which is not a time any traces were recorded at: "+ + "give a time the agent was running", + field, value) +} + +// TraceWindowEndsBeforeItStarts reports a window that can hold no traces. +func TraceWindowEndsBeforeItStarts(start, end string) error { + return fmt.Errorf( + "source.end_time %q is not after source.start_time %q, "+ + "so the window holds no traces", + end, start) +} + +// TraceWindowOverSpecified reports a window declared twice over. +// +// lookback_hours measures back from where the window closes and start_time is +// an absolute bound, so a file carrying both does not say which was meant. +func TraceWindowOverSpecified() error { + return errors.New( + "source declares both start_time and lookback_hours, which are two ways " + + "of saying where the window opens: keep one") +} + +// NegativeLookbackHours reports a lookback that is not a length. +func NegativeLookbackHours(hours int) error { + return fmt.Errorf( + "source.lookback_hours is %d, and how far back to look cannot be "+ + "negative: give the hours to look back", + hours) +} + +// LookbackTooLarge reports a lookback beyond the span a window may cover. +func LookbackTooLarge(hours, limit int) error { + return fmt.Errorf( + "source.lookback_hours is %d, which is beyond the %d hours a window can "+ + "reach back: give a shorter lookback, or replace it with a start_time", + hours, limit) +} + +// MaxTracesUnusable reports a negative cap written into the file. +// +// The flag that writes it is already guarded; this catches the file being +// edited afterwards, where a negative value is sent as-is and the run comes +// back empty. +func MaxTracesUnusable(maxTraces int) error { + return fmt.Errorf( + "source.max_traces is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTraces) +} + +// SourceFieldsNotRead reports fields the declared source type ignores. +func SourceFieldsNotRead(sourceType string, fields []string) error { + if len(fields) == 1 { + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove it, or change the type to one that does", + fields[0], sourceType) + } + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove them, or change the type to one that does", + strings.Join(fields, ", "), sourceType) +} + +// MaxTurnsUnusable reports a turn cap a run could not apply. +func MaxTurnsUnusable(maxTurns int) error { + return fmt.Errorf( + "source.max_turns is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTurns) +} + +// LookbackReachesTooFarBack reports a lookback that lands on an unusable start. +func LookbackReachesTooFarBack(hours int) error { + return fmt.Errorf( + "source.lookback_hours is %d, which opens the window before any trace "+ + "was recorded: give a shorter lookback", + hours) +} + +// AmbiguousJudgeModel reports several deployments where only one can be used. +func AmbiguousJudgeModel(models []string) error { + return fmt.Errorf( + "this project declares more than one model deployment (%s), so "+ + "--judge-model says which the graders judge with", strings.Join(models, ", ")) +} + +// SelectJudgeModelPrompt asks which deployment the graders judge with. +func SelectJudgeModelPrompt() string { + return "Select the model deployment the graders judge with:" +} + +// SelectEvalPrompt asks which of the declared evals a command means. +func SelectEvalPrompt() string { + return "Select the eval to use:" +} + +// SelectingJudgeModel reports a failed judge model prompt. +func SelectingJudgeModel(err error) error { + return fmt.Errorf("selecting a judge model deployment: %w", err) +} + +// UsingTraceSource reports a scaffold that reads production traces. +// +// Naming Application Insights is a claim about the project, so it is only made +// when a connection was actually found. `init` makes no service calls and +// cannot verify one it did not see. +func UsingTraceSource(connected bool) string { + if connected { + return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", doneMark) + } + return fmt.Sprintf( + "%s Using data source: traces. No Application Insights connection is recorded "+ + "in this environment, so the run finds rows only if the project has one\n", + doneMark) +} + +// JudgeModelDeployment reports the deployment the graders will judge with. +func JudgeModelDeployment(model string) string { + return fmt.Sprintf("%s Judge model deployment: %s\n", doneMark, model) +} + +// GradingWith reports the evaluators the scaffold settled on. +// +// Omitting --evaluator picks them, so without this the one thing `init` decided +// on the reader's behalf is the one thing it does not mention. +func GradingWith(evaluators []string) string { + return fmt.Sprintf("%s Grading with: %s\n", doneMark, strings.Join(evaluators, ", ")) +} + +// createdHeading opens the list of what a scaffold wrote. +func createdHeading() string { + return "\nCreated\n" +} + +// ScaffoldHeading opens the list of what a scaffold wrote. `init` appends to an +// existing configuration rather than replacing it, and a reader who sees +// "Created" over a file they already had reasonably fears it was overwritten. +func ScaffoldHeading(existed bool) string { + if existed { + return "\nUpdated\n" + } + return createdHeading() +} + +// createdConfigLine names the configuration a scaffold wrote. +func createdConfigLine(configPath string) string { + return fmt.Sprintf(" %-33s evaluation configuration\n", configPath) +} + +// ScaffoldConfigLine names the configuration a scaffold wrote or added to. +func ScaffoldConfigLine(configPath string, existed bool) string { + if existed { + return fmt.Sprintf(" %-33s evaluation configuration (eval added)\n", configPath) + } + return createdConfigLine(configPath) +} + +// AddedServiceLine reports the eval service being added to the root config. +func AddedServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s added service '%s'\n", rootConfig, service) +} + +// AlreadyDeclaresServiceLine reports a root config that already referenced the eval. +func AlreadyDeclaresServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s already declares service '%s'\n", rootConfig, service) +} + +// FirstNextStep opens the list of commands to run after a scaffold. +func FirstNextStep(step string) string { + return fmt.Sprintf("\nNext: %s\n", step) +} + +// FurtherNextStep continues the list of commands to run after a scaffold. +func FurtherNextStep(step string) string { + return fmt.Sprintf(" %s\n", step) +} + +// CreatedCatalogFile reports a configuration created to hold a catalog entry. +func CreatedCatalogFile(configPath string) string { + return fmt.Sprintf("%s Created %s with the catalog entry\n", doneMark, filepath.ToSlash(configPath)) +} + +// AddedToCatalog reports a generated artifact recorded in the configuration. +func AddedToCatalog(kind, artifact, configPath string) string { + return fmt.Sprintf("%s Added %s %s to %s\n", doneMark, kind, artifact, filepath.ToSlash(configPath)) +} + +// ArtifactDescription names a catalogued artifact, with its version when there is one. +func ArtifactDescription(name, version string) string { + if version == "" || version == "latest" { + return fmt.Sprintf("'%s'", name) + } + return fmt.Sprintf("'%s' (version %s)", name, version) +} + +// NoEvalsDeclared reports a configuration with nothing to act on. +// +// The same sentence wherever it is reached. `generate` writes the dataset and +// evaluator it made into the catalog but declares no eval, so a `create` or a +// run straight afterwards lands here, and both need to be told the same way +// out. +func NoEvalsDeclared() error { + return errors.New( + "no eval is declared; `azd ai eval init` declares one. " + + "`generate` only adds the dataset and evaluator it made") +} + +// SeveralEvalsDeclared reports an unnamed eval where guessing would be wrong. +// The two commands that hit this name their eval differently, so neither form +// can be recommended on its own: `create` takes it as an argument, the run +// commands take --eval. +func SeveralEvalsDeclared(count int, names []string) error { + return fmt.Errorf( + "this configuration declares %d evals (%s); name the one you mean, "+ + "as an argument to `create` or with --eval on the run commands", + count, strings.Join(names, ", ")) +} + +// EvalNotDeclared reports a name the configuration does not carry. +func EvalNotDeclared(eval string, names []string) error { + // "this configuration has" with nothing after it is a sentence that stops + // mid-clause, which is what an empty list produces. + if len(names) == 0 { + return fmt.Errorf("eval %q is not declared, and this configuration declares none", eval) + } + return fmt.Errorf( + "eval %q is not declared; this configuration has %s", + eval, strings.Join(names, ", ")) +} + +// AtLeastOneEvalRequired reports it on the way to deploying. +// +// One sentence for one fact: the deploy door and the run door reach this from +// different directions and both need the same way out. +func AtLeastOneEvalRequired() error { + return NoEvalsDeclared() +} + +// EvalNameRequired reports an eval entry with no name. +func EvalNameRequired(index int) error { + return fmt.Errorf("evals[%d]: 'name' is required", index) +} + +// DuplicateEvalName reports two evals answering to the same name. +func DuplicateEvalName(index int, eval string) error { + return fmt.Errorf("evals[%d]: duplicate eval name %q", index, eval) +} + +// EvalsIdenticalApartFromName reports two evals nothing can tell apart once deployed. +func EvalsIdenticalApartFromName(index int, eval, first string) error { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + index, eval, first) +} + +// DatasetNameRequired reports a catalog entry with no name. +func DatasetNameRequired(index int) error { + return fmt.Errorf("datasets[%d]: 'name' is required", index) +} + +// DuplicateDatasetName reports two catalog entries answering to the same name. +func DuplicateDatasetName(index int, dataset string) error { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", index, dataset) +} + +// EvaluatorNameRequired reports a catalog entry with no name. +func EvaluatorNameRequired(index int) error { + return fmt.Errorf("evaluators[%d]: 'name' is required", index) +} + +// DuplicateEvaluatorName reports two catalog entries answering to the same name. +func DuplicateEvaluatorName(index int, evaluator string) error { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", index, evaluator) +} + +// BuiltinNeedsNoCatalogEntry reports a built-in declared as though it were custom. +func BuiltinNeedsNoCatalogEntry(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", index, evaluator) +} + +// EvaluatorVersionWithSource reports a pin the service would assign anyway. +func EvaluatorVersionWithSource(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", index, evaluator) +} + +// DatasetAndSourceDeclareTheSameThing reports it where there is no index. +func DatasetAndSourceDeclareTheSameThing() error { + return errors.New("`dataset` and `source` both say where rows come from; declare one") +} + +// NoEvalToValidate reports a declaration that is not there at all. +func NoEvalToValidate() error { + return errors.New("no eval declaration to check") +} + +// DatasetNotInDatasetsCatalog reports an eval naming a dataset nobody declared. +func DatasetNotInDatasetsCatalog(index int, eval, dataset string) error { + return InEvalAt(index, eval, DatasetNotDeclared(dataset)) +} + +// DatasetNotDeclared reports it where there is no index. +func DatasetNotDeclared(dataset string) error { + return fmt.Errorf("dataset %q is not in the datasets catalog", dataset) +} + +// SourceTypeMissing reports it where there is no index. +func SourceTypeMissing() error { + return errors.New("source.type is required") +} + +// SourceTypeNotSupported reports the same, where there is no index to name. +func SourceTypeNotSupported(got, traces, responses string) error { + return fmt.Errorf("source.type %q is not supported; use %q or %q", got, traces, responses) +} + +// TraceSourceNeedsAnAgent reports it where there is no index. +// +// A target names one too, unless it names a model: a deployment name matches +// no spans, so it is not an answer to whose conversations to read. +func TraceSourceNeedsAnAgent() error { + return errors.New( + "source.agent_name is required for a trace source, " + + "or declare an agent target.name") +} + +// ResponsesSourceNeedsResponseIDs reports it where there is no index. +func ResponsesSourceNeedsResponseIDs() error { + return errors.New("source.response_ids is required for a responses source") +} + +// AtLeastOneEvaluatorRequired reports an eval that scores nothing. +func AtLeastOneEvaluatorRequired(index int, eval string) error { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", index, eval) +} + +// EvaluatorFieldRequired reports an evaluators: entry with no evaluator named. +func EvaluatorFieldRequired(evalIndex, refIndex int) error { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", evalIndex, refIndex) +} + +// DuplicateCriterion reports two result rows nothing could tell apart. +func DuplicateCriterion(evalIndex, refIndex int, criterion string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + evalIndex, refIndex, criterion) +} + +// EvaluatorNotInCatalog reports a reference to an evaluator nobody declared. +func EvaluatorNotInCatalog(evalIndex, refIndex int, evaluator string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + evalIndex, refIndex, evaluator) +} + +// TargetTypeNotSupported reports it where there is no index. +func TargetTypeNotSupported(got, agent, model string) error { + return fmt.Errorf("target.type %q is not supported; use %q or %q", got, agent, model) +} + +// EvaluationLevelNotSupported reports it where there is no index. +func EvaluationLevelNotSupported(got, turn, conversation string) error { + return fmt.Errorf("evaluation_level %q is invalid; expected %q or %q", got, turn, conversation) +} + +// TraceSourceCannotReadAModelTarget reports a trace eval pointed at a deployment. +// +// Its own sentence, because the general advice is "declare an agent target", +// which here reads as an invitation to relabel the deployment -- producing a +// filter that matches no spans and a run that reports nothing. +func TraceSourceCannotReadAModelTarget(name string) error { + return fmt.Errorf( + "source.agent_name is required for a trace source: target %q is a model "+ + "deployment, and traces are recorded against an agent, not a deployment", + name) +} + +// TargetNameMissing reports it where there is no index. +func TargetNameMissing() error { + return errors.New( + "target.name is required; remove the target: to score the dataset as it stands") +} + +// AmbiguousEvalConfig reports a directory holding both configuration names. +func AmbiguousEvalConfig(current, legacy string) error { + return fmt.Errorf( + "%s and %s are both present, and azure.yaml can reference only one of them. "+ + "Keep %s and delete the other, or point the service's $ref at the one you want", + filepath.ToSlash(current), filepath.ToSlash(legacy), filepath.ToSlash(current)) +} + +// ReadingEvalConfig reports a configuration file that would not read. +func ReadingEvalConfig(path string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return noEvalConfig(path) + } + return fmt.Errorf("reading eval config %q: %w", filepath.ToSlash(path), err) +} + +// noEvalConfig reports a command run before anything scaffolded a config. +// +// The bare read failure underneath is a Windows syscall phrase about a path, +// which describes the symptom of running `create` before `init` without naming +// either command. +// +// Still unwraps to fs.ErrNotExist, because callers that tolerate an absent +// configuration — OpenEvalConfig, and the reference resolution above it — decide +// that by asking, and a nicer sentence that stopped answering would turn every +// one of those into a failure. +func noEvalConfig(path string) error { + return &missingFileError{ + msg: fmt.Sprintf( + "no eval configuration at %s; run `azd ai eval init` to scaffold one", + filepath.ToSlash(path)), + } +} + +type missingFileError struct{ msg string } + +func (e *missingFileError) Error() string { return e.msg } +func (e *missingFileError) Unwrap() error { return fs.ErrNotExist } + +// ParsingEvalConfig reports a configuration file that would not parse. +func ParsingEvalConfig(path string, err error) error { + return fmt.Errorf("parsing eval config %q: %w", filepath.ToSlash(path), err) +} + +// SerializingEvalConfig reports a configuration that would not serialize. +func SerializingEvalConfig(err error) error { + return fmt.Errorf("serializing eval config: %w", err) +} + +// WritingEvalConfig reports a configuration file that would not be written. +func WritingEvalConfig(path string, err error) error { + return fmt.Errorf("writing eval config %q: %w", filepath.ToSlash(path), err) +} + +// ErrAmbiguousAgentService reports that a target name matched more than one service. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AmbiguousAgentService reports a --target that names no single set of instructions. +func AmbiguousAgentService(agent string, matched []string) error { + return fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agent, strings.Join(matched, ", ")) +} + +// InstructionFileUnreadable reports optimize metadata pointing at a missing file. +func InstructionFileUnreadable(metadataPath, named string, err error) error { + return fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + metadataPath, named, err) +} + +// ListingTruncated reports a page walk that stopped before the end. +// +// Worth saying out loud rather than logging: a short evaluator listing resolves +// the latest version from the pages that arrived, so a truncated one can pick +// an older version and report nothing unusual. +func ListingTruncated(pages int) error { + return fmt.Errorf( + "stopped reading the listing after %d pages, so it may be incomplete", pages) +} + +// ServiceRefPointsElsewhere reports an existing service entry wired to a +// different configuration than the one just scaffolded. +func ServiceRefPointsElsewhere(serviceName, have, want string) error { + return fmt.Errorf( + "service %q already points at %s, and the configuration just written is "+ + "%s; point the service's $ref at the one you want, or scaffold with "+ + "--path %s", serviceName, have, want, have) +} + +// InstructionFileOutsideProject reports metadata pointing outside the project. +// +// The pointer is read from a file in the checkout, so it is only as trustworthy +// as the checkout: an absolute path or one climbing out with `..` would read +// something the project does not contain and send it on as agent instructions. +func InstructionFileOutsideProject(metadataPath, named string) error { + return fmt.Errorf( + "%s names instruction_file %q, which is outside the project; "+ + "name a path inside it", metadataPath, named) +} + +// FromNotASource reports a --from value the generation service has no path for. +func FromNotASource(from string, sources []string) error { + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(sources, ", ")) +} + +// ConfigLockUnavailable reports a config lock that could not be taken. +// +// Not fatal, and said out loud for that reason: the work goes ahead unlocked, +// so a lost update afterwards has no other explanation on record. +func ConfigLockUnavailable(evalDir string, err error) error { + if err == nil { + return fmt.Errorf( + "another process is still updating %s, so this update is not "+ + "serialized against it", filepath.ToSlash(evalDir)) + } + return fmt.Errorf( + "could not lock %s, so this update is not serialized against other "+ + "processes: %w", filepath.ToSlash(evalDir), err) +} + +// InvalidNextLink reports a pagination link the service sent that will not parse. +func InvalidNextLink(link string, err error) error { + return fmt.Errorf("invalid nextLink %q: %w", link, err) +} + +// NextLinkOffOrigin reports a pagination link pointing somewhere other than the +// project endpoint. Following it would send the caller's token to that host. +func NextLinkOffOrigin(origin string) error { + return fmt.Errorf("refusing to follow nextLink to %s: it is not the project endpoint", origin) +} + +// PageLinkLeftTheService reports a paging link pointing somewhere else. +// +// The link arrives in a response body and this client sends an Authorization +// header, so following one to another host would send the token there. +func PageLinkLeftTheService(expected, got string) error { + return fmt.Errorf( + "the service returned a paging link for %q while this client is "+ + "talking to %q, so it was not followed", got, expected) +} + +// SampleSizeOutOfRange reports a row count the generation service would reject. +func SampleSizeOutOfRange(min, max, got int) error { + return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) +} + +// MaxSamplesNegative reports a declared row cap below zero. +// +// Anything not above zero reads as "no cap", so this used to send the whole +// dataset to a run that is billed per row -- the opposite of what a cap asks +// for, and silent. +func MaxSamplesNegative(got int) error { + return fmt.Errorf( + "max_samples cannot be negative, got %d. "+ + "Remove it to send every row, or set the number of rows to send", got) +} + +// NegativeMaxSamplesFlag reports the same thing given on the command line. +func NegativeMaxSamplesFlag(got int) error { + return fmt.Errorf( + "--max-samples cannot be negative, got %d. "+ + "Omit it to send every row, or give the number of rows to send", got) +} + +// FlagDoesNotApply reports a flag given to a generate that produces nothing it +// could affect. +// +// Each of these is read while building one kind of artifact and ignored while +// building the other, so given for the wrong one they were accepted and +// dropped: `--evaluator --max-samples 50` produced a rubric and said nothing +// about the 50. +func FlagDoesNotApply(flag, narrowedBy string) error { + return fmt.Errorf( + "--%s has no effect on what %s generates. "+ + "Drop --%s, or drop %s to generate both", flag, narrowedBy, flag, narrowedBy) +} + +// NegativeTraceDays reports a trace window below zero. +// +// Zero already means "do not read traces", so a negative value has nothing +// left to mean; it used to be accepted and treated as zero, which silently +// produced a rubric with none of the trace seeding that was asked for. +func NegativeTraceDays(got int) error { + return fmt.Errorf( + "--trace-days cannot be negative, got %d. "+ + "Use 0 to seed the rubric from no traces, or the number of days to read", got) +} + +// OutputDirNeedsTheWait reports an output directory that nothing will be +// written to. +// +// --no-wait returns as soon as the job is submitted, so there is no artifact +// to place. Accepting both left the caller waiting for a file that was never +// coming. +func OutputDirNeedsTheWait() error { + return errors.New( + "--output-dir has nothing to write to with --no-wait, which returns " + + "before the artifact exists. Drop --no-wait, or collect the " + + "artifact later with `azd ai eval job show`") +} + +// EndpointEmpty reports a project endpoint given as blank. +func EndpointEmpty() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must not be empty", + "provide a Foundry project endpoint URL "+ + "(e.g. https://.services.ai.azure.com/api/projects/)", + ) +} + +// EndpointUnparseable reports a project endpoint that is not a URL. +func EndpointUnparseable(err error) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("invalid project endpoint URL: %v", err), + "provide a valid https:// Foundry project endpoint URL", + ) +} + +// EndpointNotHTTPS reports a project endpoint on the wrong scheme. +func EndpointNotHTTPS() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must use https", + "provide an https:// URL", + ) +} + +// EndpointNotFoundryHost reports a project endpoint pointing somewhere else. +func EndpointNotFoundryHost(host, suffix string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf( + "project endpoint host %q is not a recognized Foundry host (*%s)", + host, suffix, + ), + "the host must end with "+suffix, + ) +} + +// EndpointHasPort reports a project endpoint carrying an explicit port. +func EndpointHasPort(host string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("project endpoint host %q must not include a port", host), + "remove the explicit port from the URL", + ) +} + +// NoEndpoint reports a project endpoint that no source could supply. +func NoEndpoint() error { + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint resolved", + "persist a workspace default with `azd ai project set `, "+ + "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ + "in the active azd environment, "+ + "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", + ) +} + +// ProjectContextClient reports the config helper failing to build. +func ProjectContextClient(err error) error { + return fmt.Errorf("getProjectContext: %w", err) +} + +// ProjectContextRead reports the persisted project context failing to read. +func ProjectContextRead(err error) error { + return fmt.Errorf("getProjectContext: failed to read config: %w", err) +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + +// Warning reports a problem that is not worth failing the command over. +func Warning(err error) string { + return fmt.Sprintf("warning: %v\n", err) +} + +// PortalLink closes a detail view with the asset's portal URL. +func PortalLink(url string) string { + return fmt.Sprintf("Portal: %s\n", url) +} + +// FlagRequired reports a value the command needs and cannot settle itself. +// +// It used to add "(running with --no-prompt)", which was untrue at every call +// site: none of them prompts, so the parenthetical named a flag the caller had +// not passed and implied that dropping it would make the command ask. +func FlagRequired(name string) error { + return fmt.Errorf("--%s is required", name) +} + +// Creating reports a directory or file that could not be created. +func Creating(path string, err error) error { + return fmt.Errorf("creating %q: %w", filepath.ToSlash(path), err) +} + +// Serializing reports a value that could not be written out. +func Serializing(path string, err error) error { + return fmt.Errorf("serializing %q: %w", filepath.ToSlash(path), err) +} + +// Writing reports a file that could not be written. +func Writing(path string, err error) error { + return fmt.Errorf("writing %q: %w", filepath.ToSlash(path), err) +} + +// ReadingPath reports a file or directory that could not be read. +func ReadingPath(path string, err error) error { + return fmt.Errorf("reading %s: %w", path, err) +} + +// quoteList renders names as a readable "a", "b" and "c". +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +// pluralColumns agrees with however many columns are missing. +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" +} + +// --------------------------------------------------------------------------- +// Talking to the service +// --------------------------------------------------------------------------- + +// InvalidEndpointURL reports a client built on an endpoint that will not parse. +func InvalidEndpointURL(err error) error { + return fmt.Errorf("invalid endpoint URL: %w", err) +} + +// InvalidRequestPath reports a request path that will not parse. +func InvalidRequestPath(path string, err error) error { + return fmt.Errorf("invalid request path %q: %w", path, err) +} + +// CreatingRequest reports a request that could not be built. +func CreatingRequest(err error) error { + return fmt.Errorf("failed to create request: %w", err) +} + +// MarshalingRequest reports a request body that would not serialize. +func MarshalingRequest(err error) error { + return fmt.Errorf("failed to marshal request: %w", err) +} + +// SettingRequestBody reports a request body that would not attach. +func SettingRequestBody(err error) error { + return fmt.Errorf("failed to set request body: %w", err) +} + +// RequestFailed reports a request that never reached an answer. +// +// A credential that cannot mint a token fails here rather than as a 401, and +// the SDK's own text for it names neither azd nor the way out. isCredentialFailure +// decides which is which; see it for how. +// +// The hint is in the message as well as the suggestion because the suggestion +// is not rendered on every surface, and it offers a retry first: this call +// shells out to `azd auth token`, which has been seen to fail transiently +// against a login that was perfectly valid -- measured once at over 70 seconds, +// long enough to lose to a deadline. +func RequestFailed(err error) error { + if isCredentialUnavailable(err) { + // Not an expired login, and `azd auth login` cannot be run to fix it. + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf( + "could not get a token for the Foundry project because azd itself "+ + "could not be run: %v", err), + "check that `azd` is installed and on PATH") + } + if isCredentialFailure(err) { + return exterrors.Auth( + exterrors.CodeLoginExpired, + fmt.Sprintf( + "could not get a token for the Foundry project: %v. "+ + "Try again; if it keeps failing, run `azd auth login`", err), + "try the command again, then `azd auth login` if it keeps failing") + } + return fmt.Errorf("HTTP request failed: %w", err) +} + +// isCredentialUnavailable reports the credential never having run at all, as +// opposed to running and being refused. +// +// azidentity's credentialUnavailableError is unexported, so this matches the +// two messages it carries for that case. Worth separating because the answer +// to both is not `azd auth login` -- you cannot log in with a tool that is not +// on PATH. +func isCredentialUnavailable(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "executable not found on path") || + strings.Contains(text, "is not recognized") +} + +// ServiceRefused turns an unauthorized answer into one that says what to do. +// Every other status is left as the service reported it. +func ServiceRefused(status int, err error) error { + if status == http.StatusUnauthorized || status == http.StatusForbidden { + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf( + "the Foundry project refused the request (HTTP %d): %v. "+ + "Run `azd auth login`, and check you have access to this project", + status, err), + "run `azd auth login`, and check you have access to this project") + } + return err +} + +// isCredentialFailure reports whether the request failed because no token could +// be minted, rather than for any of the other reasons a request fails. +// +// Decided on the SDK's own error types. This used to also match the phrase +// "failed to acquire a token" anywhere in the text, which any error is free to +// contain -- a service that could not acquire a token bucket lease was told its +// login had expired and to run `azd auth login`. +// +// The credential names stay as a fallback because credentialUnavailableError is +// unexported: a credential that never ran can only be recognized by the name it +// puts in its own message. +func isCredentialFailure(err error) bool { + if err == nil { + return false + } + + var authFailed *azidentity.AuthenticationFailedError + var authRequired *azidentity.AuthenticationRequiredError + if errors.As(err, &authFailed) || errors.As(err, &authRequired) { + return true + } + + text := err.Error() + for _, credential := range []string{ + "AzureDeveloperCLICredential", + "DefaultAzureCredential", + } { + if strings.Contains(text, credential) { + return true + } + } + return false +} + +// ReadingResponseBody reports a response that could not be read. +func ReadingResponseBody(err error) error { + return fmt.Errorf("failed to read response body: %w", err) +} + +// ParsingResponse reports a response that could not be parsed. +func ParsingResponse(err error) error { + return fmt.Errorf("failed to parse response: %w", err) +} + +// ParsingNumber reports a numeric field that did not arrive as a number. +func ParsingNumber(data string, err error) error { + return fmt.Errorf("parsing number %s: %w", data, err) +} + +// InvalidContainerURI reports a storage URI the service handed back unusable. +func InvalidContainerURI(err error) error { + return fmt.Errorf("invalid container SAS URI: %w", err) +} + +// CreatingUploadRequest reports the blob upload request failing to build. +func CreatingUploadRequest(err error) error { + return fmt.Errorf("failed to create upload request: %w", err) +} + +// UploadingBlobFailed reports the blob upload never reaching an answer. +func UploadingBlobFailed(err error) error { + return fmt.Errorf("failed to upload blob: %w", err) +} + +// BlobUploadStatus reports storage refusing the upload. +func BlobUploadStatus(status int, body string) error { + return fmt.Errorf("blob upload failed with status %d: %s", status, body) +} + +// CreatingDownloadRequest reports the dataset download request failing to build. +func CreatingDownloadRequest(err error) error { + return fmt.Errorf("failed to create download request: %w", err) +} + +// DownloadingDatasetBlob reports the dataset download never reaching an answer. +func DownloadingDatasetBlob(err error) error { + return fmt.Errorf("failed to download dataset from blob: %w", err) +} + +// BlobDownloadStatus reports storage refusing the download. +func BlobDownloadStatus(status int) error { + return fmt.Errorf("blob download failed with status %d", status) +} + +// ReadingDatasetContent reports a downloaded dataset that could not be read. +func ReadingDatasetContent(err error) error { + return fmt.Errorf("failed to read dataset content: %w", err) +} + +// CreatingListRequest reports the container listing request failing to build. +func CreatingListRequest(err error) error { + return fmt.Errorf("failed to create list request: %w", err) +} + +// ListingContainerBlobs reports the container listing never reaching an answer. +func ListingContainerBlobs(err error) error { + return fmt.Errorf("failed to list container blobs: %w", err) +} + +// ContainerListStatus reports storage refusing the listing. +func ContainerListStatus(status int) error { + return fmt.Errorf("container list failed with status %d", status) +} + +// ReadingListResponse reports a container listing that could not be read. +func ReadingListResponse(err error) error { + return fmt.Errorf("failed to read list response: %w", err) +} + +// CreatingBlobDownloadRequest reports the blob download request failing to build. +func CreatingBlobDownloadRequest(err error) error { + return fmt.Errorf("failed to create blob download request: %w", err) +} + +// DownloadingBlob reports one blob's download never reaching an answer. +func DownloadingBlob(err error) error { + return fmt.Errorf("failed to download blob: %w", err) +} + +// BlobDownloadStatusFor reports storage refusing one named blob. +func BlobDownloadStatusFor(status int, blobName string) error { + return fmt.Errorf("blob download failed with status %d for %s", status, blobName) +} + +// ReadingBlobContent reports a downloaded blob that could not be read. +func ReadingBlobContent(err error) error { + return fmt.Errorf("failed to read blob content: %w", err) +} + +// ParsingProjectResourceID reports a project ARM id that will not parse. +func ParsingProjectResourceID(err error) error { + return fmt.Errorf("failed to parse project resource ID: %w", err) +} + +// EncodingSubscriptionID reports a subscription that would not encode for a URL. +func EncodingSubscriptionID(err error) error { + return fmt.Errorf("failed to encode subscription ID: %w", err) +} + +// NotAFoundryProjectResourceID reports an ARM id that names something else. +func NotAFoundryProjectResourceID(resourceID string) error { + return fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + resourceID) +} + +// InvalidSubscriptionID reports a subscription id that is not a GUID. +func InvalidSubscriptionID(err error) error { + return fmt.Errorf("invalid subscription ID format: %w", err) +} + +// CouldNotReadAgentForModel reports a target agent whose deployment could not +// be read, leaving generation without a default model. +func CouldNotReadAgentForModel(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so there is no "+ + "deployment to default to; pass --generation-model\n", agent) + } + return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go new file mode 100644 index 00000000000..985a9e95b01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "net/http" + "strings" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// suggestionOf reads the way out azd renders beneath the message. It is a +// structured field, not part of Error(), so asserting on the text alone would +// pass whether or not the suggestion survived. +func suggestionOf(t *testing.T, err error) string { + t.Helper() + var local *azdext.LocalError + require.ErrorAs(t, err, &local, "expected a structured local error") + return local.Suggestion +} + +// A stale login fails when the credential tries to mint a token, not as a 401, +// and the SDK's text for it mentions neither azd nor logging in. This was seen +// live: "HTTP request failed: AzureDeveloperCLICredential: exit status 1". +func TestRequestFailed_ClassifiesAStaleLogin(t *testing.T) { + err := RequestFailed(errors.New("AzureDeveloperCLICredential: exit status 1")) + + require.Error(t, err) + assert.Contains(t, suggestionOf(t, err), "azd auth login", + "a token failure has to name the way out") + assert.Contains(t, err.Error(), "exit status 1", "the cause still has to be visible") +} + +// Anything that is not a credential problem keeps the cause it came with. +func TestRequestFailed_PassesOtherFailuresThrough(t *testing.T) { + cause := errors.New("dial tcp: connection refused") + + err := RequestFailed(cause) + + require.Error(t, err) + assert.ErrorIs(t, err, cause, "the cause has to survive wrapping") + assert.NotContains(t, err.Error(), "azd auth login") +} + +func TestServiceRefused_ClassifiesUnauthorized(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := ServiceRefused(status, errors.New("the service said no")) + + require.Error(t, err) + assert.Contains(t, suggestionOf(t, err), "azd auth login", + "HTTP %d has to name the way out", status) + } +} + +// A 404 is not an auth problem and must not be reported as one. +func TestServiceRefused_LeavesOtherStatusesAlone(t *testing.T) { + cause := errors.New("not found") + + err := ServiceRefused(http.StatusNotFound, cause) + + assert.Same(t, cause, err, "a non-auth status is returned untouched") +} + +// The wait ending is not the run failing, so the line says which stopped. +func TestWaitBudgetSpent_SaysTheRunIsStillGoing(t *testing.T) { + line := WaitBudgetSpent("evalrun_1", 0) + + assert.Contains(t, line, "evalrun_1") + assert.Contains(t, strings.ToLower(line), "still going") +} + +// An interrupted wait has to name the run, or it is lost to whoever stopped it. +func TestWaitInterrupted_NamesTheRunAndTheWayBack(t *testing.T) { + err := WaitInterrupted("evalrun_2", errors.New("context canceled")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "evalrun_2") + assert.Contains(t, err.Error(), "azd ai eval run show") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go new file mode 100644 index 00000000000..29b97ba637e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "fmt" + "io/fs" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// configPath and friends build the separator this OS actually uses. Hardcoding +// a backslash only exercises the escaping on Windows: filepath.ToSlash rewrites +// the platform separator, so on Linux a literal backslash is left alone -- it is +// part of a filename there, not a separator. +var ( + testConfigPath = filepath.Join("evals", "azure.eval.yaml") + testDatasetDir = filepath.Join("evals", "datasets") + testDatasetPath = filepath.Join("evals", "datasets", "d.jsonl") + testEvaluatorPath = filepath.Join("evals", "evaluators", "e.json") + testInstructions = filepath.Join("docs", "instructions.md") + testOutDir = filepath.Join("evals", "out") + testOutPath = filepath.Join("evals", "out", "x.json") +) + +// Running `create` before `init` is the first thing anyone does wrong, and the +// bare read failure underneath is a Windows syscall phrase naming neither +// command. +// +// It must still unwrap to fs.ErrNotExist: the callers that tolerate an absent +// configuration decide that by asking, so a nicer sentence that stopped +// answering would turn every one of those into a failure. +func TestNoEvalConfigStaysDetectable(t *testing.T) { + err := ReadingEvalConfig(testConfigPath, fmt.Errorf("open x: %w", fs.ErrNotExist)) + + assert.Contains(t, err.Error(), "no eval configuration at evals/azure.eval.yaml") + assert.Contains(t, err.Error(), "azd ai eval init") + assert.NotContains(t, err.Error(), "The system cannot find") + assert.True(t, errors.Is(err, fs.ErrNotExist), + "callers tolerate an absent config by asking, so it has to keep answering") + + other := ReadingEvalConfig(testConfigPath, errors.New("permission denied")) + assert.Contains(t, other.Error(), "reading eval config") + assert.False(t, errors.Is(other, fs.ErrNotExist)) +} + +// as "evals\\azure.eval.yaml". A reader who copies that into a shell gets a path +// that does not exist, which is the opposite of what naming the file is for. +func TestPathsInMessagesStayCopyable(t *testing.T) { + boom := errors.New("no such file") + + for _, err := range []error{ + ReadingEvalConfig(testConfigPath, boom), + ParsingEvalConfig(testConfigPath, boom), + WritingEvalConfig(testConfigPath, boom), + ReadingFromFile(testDatasetPath, boom), + FromFileMustBeJSONL(testDatasetDir), + DatasetSource(testDatasetPath, boom), + EvaluatorSource(testEvaluatorPath, boom), + DatasetFileEmpty(testDatasetPath), + ReadingInstructionFile(testInstructions, boom), + InstructionFileEmpty(testInstructions), + Hashing(testEvaluatorPath, boom), + Creating(testOutDir, boom), + Serializing(testOutPath, boom), + Writing(testOutPath, boom), + } { + got := err.Error() + assert.NotContains(t, got, `\\`, "a doubled separator is not a path anyone can use: %s", got) + assert.Truef(t, strings.Contains(got, "/"), "the path should read with forward slashes: %s", got) + } +} + +// The separator conversion only has anything to convert on Windows: on Linux a +// backslash is an ordinary character in a filename, and filepath.ToSlash leaves +// it alone -- correctly. Without this case the suite would stay green on Linux +// with every filepath.ToSlash call deleted. +func TestWindowsSeparatorsRenderAsForwardSlashes(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("filepath.ToSlash only rewrites the platform separator") + } + + got := ReadingEvalConfig(`evals\azure.eval.yaml`, errors.New("no such file")).Error() + + assert.Contains(t, got, "evals/azure.eval.yaml") + assert.NotContains(t, got, `\\`, + "%%q escapes a Windows separator, so the path stops being copyable: %s", got) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go new file mode 100644 index 00000000000..1a30d4c4245 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// JudgeModelRequired shipped ending at "to read one from" -- the half of the +// sentence naming the way out never rendered. It is the first error a new user +// hits, and nothing caught it because no test asserted the rendered string: +// checking the error type or a prefix cannot see a missing tail. +func TestJudgeModelRequiredNamesBothPlacesItLooked(t *testing.T) { + msg := JudgeModelRequired().Error() + + assert.Contains(t, msg, "--judge-model", "the flag is the whole of the way out") + assert.Contains(t, msg, "AZURE_AI_MODEL_DEPLOYMENT_NAME", + "the other place a deployment is read from has to be named") + assert.NotContains(t, msg, "to read one from", "the sentence must not stop mid-clause") + assert.False(t, strings.HasSuffix(strings.TrimRight(msg, " "), ":"), + "a trailing colon promises a clause that never arrives: %q", msg) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go new file mode 100644 index 00000000000..b9c392f0dba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages_test + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// importPath is what a file has to import for its `x.Name` selectors to count. +const importPath = "azureaieval/internal/messages" + +// Every sentence in this package has to be spoken by something a user runs. +// +// The package exists so the whole voice of the CLI can be read in one sitting. +// A constructor nobody calls is a sentence in that reading which no user will +// ever see, and it is worse than clutter: three times now, consolidating two +// ways of saying one thing has left the losing wording behind -- still +// compiling, still reviewed, ready to be picked up by someone who finds it and +// assumes it is live. `unused` cannot catch them, because they are exported. +// +// A reference from a test does not count. A message that only a test calls is +// still a sentence no user sees, and counting tests would let the deleted +// wording be kept alive by the test written for it, which is exactly what +// happened to `evalIDKeys`. +func TestEveryMessageIsSpokenBySomethingAUserRuns(t *testing.T) { + root := moduleRoot(t) + pkgDir := filepath.Join(root, "internal", "messages") + + declared := exported(t, pkgDir) + require.NotEmpty(t, declared, "the package should declare messages") + + spoken := referenced(t, root, pkgDir, false) + fromTests := referenced(t, root, pkgDir, true) + + var silent, testOnly []string + for name := range declared { + switch { + case spoken[name]: + case fromTests[name]: + testOnly = append(testOnly, name) + default: + silent = append(silent, name) + } + } + + require.Empty(t, silent, + "messages nothing calls: delete them, or call them.\n%s", strings.Join(silent, "\n")) + require.Empty(t, testOnly, + "messages only a test calls, so no user ever sees them:\n%s", strings.Join(testOnly, "\n")) +} + +// moduleRoot walks up from this package to the directory holding go.mod. +func moduleRoot(t *testing.T) string { + t.Helper() + dir, err := filepath.Abs(".") + require.NoError(t, err) + for { + // Stat rather than Glob: Glob reads its argument as a pattern, so a + // checkout path containing a bracket answers ErrBadPattern and the + // walk blames the repo layout for a path this test chose to build. + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + require.NotEqual(t, dir, parent, "no go.mod above the messages package") + dir = parent + } +} + +// exported collects the package-level exported names declared in dir. +// +// Functions, and the vars and consts beside them: a marker nobody prints +// is as unspoken as a constructor nobody calls. +func exported(t *testing.T, dir string) map[string]bool { + t.Helper() + names := map[string]bool{} + + for _, file := range parseDir(t, dir) { + if strings.HasSuffix(file.path, "_test.go") { + continue + } + for _, decl := range file.ast.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + // A method is reached through its type, not by name here. + if d.Recv == nil && d.Name.IsExported() { + names[d.Name.Name] = true + } + case *ast.GenDecl: + for _, spec := range d.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for _, ident := range value.Names { + if ident.IsExported() { + names[ident.Name] = true + } + } + } + } + } + } + return names +} + +// referenced collects the names selected off this package elsewhere in the +// module, from test files or from the rest of it. +func referenced(t *testing.T, root, pkgDir string, inTests bool) map[string]bool { + t.Helper() + used := map[string]bool{} + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // An entry this test cannot read says nothing about messages, and + // failing the invariant over it would report the wrong problem. + return nil //nolint:nilerr // unreadable entries are not this test's business + } + if d.IsDir() { + // The package's own internal tests call these bare, so they + // contribute no selectors; an external one beside this file would, + // and would make the check satisfy itself. + if path == pkgDir { + return fs.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") != inTests { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly|parser.SkipObjectResolution) + if err != nil { + return nil //nolint:nilerr // a file that will not parse is the compiler's to report + } + local, ok := localName(file) + if !ok { + return nil + } + full, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution) + if err != nil { + return nil //nolint:nilerr // a file that will not parse is the compiler's to report + } + ast.Inspect(full, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + // Matched against the name this file bound the import to, so an + // aliased import counts and an unrelated `messages` identifier + // does not. + if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == local { + used[sel.Sel.Name] = true + } + return true + }) + return nil + }) + require.NoError(t, err) + return used +} + +// localName is the name a file binds this package to, if it imports it. +func localName(file *ast.File) (string, bool) { + for _, imp := range file.Imports { + if strings.Trim(imp.Path.Value, `"`) != importPath { + continue + } + if imp.Name != nil { + // A dot import puts the names in scope unqualified, which this + // test cannot follow; nothing in the module does it. + if imp.Name.Name == "." || imp.Name.Name == "_" { + return "", false + } + return imp.Name.Name, true + } + return "messages", true + } + return "", false +} + +type parsedFile struct { + path string + ast *ast.File +} + +func parseDir(t *testing.T, dir string) []parsedFile { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + var files []parsedFile + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + continue + } + path := filepath.Join(dir, entry.Name()) + // ParseDir is deprecated as of Go 1.22, and reading the directory + // here costs nothing. + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution) + require.NoError(t, err) + files = append(files, parsedFile{path: path, ast: file}) + } + return files +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go new file mode 100644 index 00000000000..6a121f289dd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// `init` printed "Using data source: traces (Application Insights)" whether or +// not the project had one, so a scaffold that could not produce a sample was +// reported with a green check. init makes no service calls, so it cannot +// verify a connection it never saw. +func TestTraceSourceOnlyClaimsAConnectionItFound(t *testing.T) { + connected := UsingTraceSource(true) + assert.Contains(t, connected, "Application Insights") + + unverified := UsingTraceSource(false) + assert.Contains(t, unverified, "traces", + "the source is still what was chosen") + assert.NotContains(t, unverified, "(Application Insights)", + "naming the connection asserts something nobody checked") + assert.Truef(t, strings.Contains(unverified, "only if"), + "the reader has to learn the run may find no rows: %s", unverified) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go new file mode 100644 index 00000000000..3a6ef0f7828 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func blobPage(marker string, names ...string) string { + var body strings.Builder + body.WriteString(``) + for _, n := range names { + body.WriteString(fmt.Sprintf(`%s`, n)) + } + body.WriteString(`` + marker + ``) + return body.String() +} + +// DownloadDatasetContent falls back to listing the container and taking the +// first .jsonl by name, so a container answered one page at a time could report +// no file, or a different one, depending on where the page happened to end. +func TestListContainerBlobsFollowsTheMarker(t *testing.T) { + var markers []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m := r.URL.Query().Get("marker") + markers = append(markers, m) + w.Header().Set("Content-Type", "application/xml") + switch m { + case "": + fmt.Fprint(w, blobPage("m1", "a.jsonl", "b.jsonl")) + case "m1": + fmt.Fprint(w, blobPage("m2", "c.jsonl")) + default: + fmt.Fprint(w, blobPage("", "d.jsonl")) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + names, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container?sig=redacted") + + require.NoError(t, err) + assert.Equal(t, []string{"a.jsonl", "b.jsonl", "c.jsonl", "d.jsonl"}, names) + assert.Equal(t, []string{"", "m1", "m2"}, markers, + "each request has to carry the marker the previous page returned") +} + +// An empty NextMarker is the last page, which is what this did before it could +// see the marker at all. +func TestListContainerBlobsStopsWithoutAMarker(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, blobPage("", "only.jsonl")) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + names, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container") + + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Equal(t, []string{"only.jsonl"}, names) +} + +// A marker that repeats itself would otherwise spin until the page bound. +func TestListContainerBlobsStopsOnARepeatedMarker(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, blobPage("stuck", "same.jsonl")) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + _, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container") + + require.NoError(t, err) + assert.Equal(t, 2, calls, "the first request, then the marker once") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go new file mode 100644 index 00000000000..f4059a61779 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sasSecret is the signature a storage SAS carries in its query string. +const constructionSASSecret = "SIGNATUREVALUETHATMUSTNOTAPPEAR" + +// A URL the parser refuses, still carrying a SAS. Building a request from it +// fails before any transport error can happen, which is the path the redaction +// work missed: Do's failures were wrapped and NewRequest's were not. +func malformedSASURL() string { + return "https://acct.blob.core.windows.net/c/d.jsonl\x7f?sig=" + constructionSASSecret +} + +func constructionClient(t *testing.T) *DatasetClient { + t.Helper() + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline("https://example.invalid", pipeline) +} + +// The premise: the raw error names the URL, so returning it unwrapped hands the +// signature to whoever reads the message or the debug log. +func TestRequestConstructionErrorsDoNotCarryTheSAS(t *testing.T) { + raw := malformedSASURL() + require.Contains(t, raw, constructionSASSecret, "the fixture has to carry a secret to leak") + + client := constructionClient(t) + ctx := context.Background() + + cases := []struct { + name string + call func() error + }{ + {"DownloadDataset", func() error { _, err := client.DownloadDataset(ctx, raw); return err }}, + {"DownloadBlob", func() error { _, err := client.DownloadBlob(ctx, raw, "d.jsonl"); return err }}, + {"UploadBlob", func() error { return client.UploadBlob(ctx, raw, "d.jsonl", []byte("{}")) }}, + {"ListContainerBlobs", func() error { _, err := client.ListContainerBlobs(ctx, raw); return err }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + + require.Error(t, err, "a URL the parser refuses has to fail") + assert.NotContains(t, err.Error(), constructionSASSecret, + "the signature reached the caller through %s", tc.name) + assert.NotContains(t, strings.ToLower(err.Error()), "sig=", + "even the parameter name should not survive") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go new file mode 100644 index 00000000000..001c3ab50a1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A dataset's URI points at either the blob or the container holding it, +// depending on how it was created, and nothing in the payload says which: +// isSingleFile is true either way. Uploaded datasets end in the file name; +// generated ones end in the container. Downloading a container returns 409. +func TestLooksLikeBlobURI(t *testing.T) { + uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" + generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" + + require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") + require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") +} + +// A SAS token on the URI must not change the answer. +func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { + require.True(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) +} + +// An evaluation dataset is JSONL, so that is preferred when a container holds +// more than one file. +func TestPickDatasetBlobPrefersJSONL(t *testing.T) { + require.Equal(t, "data.jsonl", + pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) + require.Equal(t, "data.JSONL", + pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") +} + +// With nothing recognizable, any real file beats returning nothing. +func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { + require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) + require.Empty(t, pickDatasetBlob([]string{"folder/"})) + require.Empty(t, pickDatasetBlob(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go new file mode 100644 index 00000000000..822ad65c627 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testAPIVersion = "2025-11-15-preview" + +// blobListing is the shape Azure Blob Storage answers a container list with. +func blobListing(names ...string) string { + var b strings.Builder + b.WriteString(``) + for _, n := range names { + b.WriteString("" + n + "") + } + b.WriteString(``) + return b.String() +} + +// storageServer stands in for both the dataset API and blob storage, recording +// what each leg of a download was asked for. +type storageServer struct { + mu sync.Mutex + + // credential is the sasUri handed back for a download, relative to the + // server's own address. + uriPath string + // blobs maps a container-relative blob name to its content. + blobs map[string]string + // directBlobStatus is the status a direct GET of uriPath answers. + directBlobStatus int + + gotListQuery url.Values + gotBlobPaths []string + gotAPIVer []string +} + +func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if v := r.URL.Query().Get("api-version"); v != "" { + s.gotAPIVer = append(s.gotAPIVer, v) + } + + switch { + case strings.HasSuffix(r.URL.Path, "/credentials"): + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine, and + // FailNow there aborts mid-response and fails whichever test is + // running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReferenceForConsumption": map[string]any{ + "credential": map[string]any{"sasUri": srv.URL + s.uriPath + "?sig=secret"}, + }, + })) + + case r.URL.Query().Get("comp") == "list": + s.gotListQuery = r.URL.Query() + names := make([]string, 0, len(s.blobs)) + for n := range s.blobs { + names = append(names, n) + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(blobListing(names...))) + + // A direct read of the credential URI itself, keyed under "". + case r.URL.Path == s.uriPath: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + if s.directBlobStatus != 0 { + w.WriteHeader(s.directBlobStatus) + return + } + _, _ = w.Write([]byte(s.blobs[""])) + + default: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.uriPath+"/")] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + return client, srv +} + +// A dataset that was uploaded names its own file, so it reads in one hop and +// the container must never be listed. +func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { + server := &storageServer{ + uriPath: "/c/rows.jsonl", + blobs: map[string]string{"": `{"query":"direct"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"direct"}`, string(data)) + assert.Nil(t, server.gotListQuery, "a blob URI needs no container listing") +} + +// A generated dataset names the container it was written into, and nothing in +// the payload says so: isSingleFile is true either way. Reading the container +// directly returns a 409, so the blob inside has to be found first. +func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { + server := &storageServer{ + uriPath: "/generated-container", + blobs: map[string]string{ + "_meta.json": `{"ignored":true}`, + "data.jsonl": `{"query":"from the container"}`, + }, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"from the container"}`, string(data), + "the JSONL is chosen over the metadata sitting beside it") + require.NotNil(t, server.gotListQuery) + assert.Equal(t, "container", server.gotListQuery.Get("restype")) + assert.Equal(t, "secret", server.gotListQuery.Get("sig"), + "the listing must keep the SAS token, or storage answers 403") +} + +// A URI can name a file and still be a container — the extension is a guess, +// not a fact. When the direct read fails the listing is the fallback, so the +// download succeeds rather than surfacing the first status. +func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { + server := &storageServer{ + uriPath: "/c/looks.jsonl", + directBlobStatus: http.StatusConflict, + blobs: map[string]string{"real.jsonl": `{"query":"found by listing"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err, "a 409 on the direct read is the container case, not a failure") + assert.Equal(t, `{"query":"found by listing"}`, string(data)) + assert.NotNil(t, server.gotListQuery) +} + +// An empty container is a dataset with nothing to read, and saying so beats +// returning empty content that looks like a dataset with no rows. +func TestDownloadDatasetContentReportsAnEmptyContainer(t *testing.T) { + server := &storageServer{uriPath: "/empty", blobs: map[string]string{}} + client, _ := server.start(t) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no downloadable file") +} + +// The URI carries no SAS of its own, so a credential that resolves to nothing +// has to be reported here rather than as an unauthorized read later. +func TestDownloadDatasetContentRequiresADownloadURI(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no download URI") +} + +// The blob name is appended to the container path, and the SAS token stays on +// the query where storage expects it. +func TestDownloadBlobKeepsTheSASToken(t *testing.T) { + var gotPath, gotSig string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSig = r.URL.Path, r.URL.Query().Get("sig") + _, _ = w.Write([]byte("rows")) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + data, err := client.DownloadBlob(context.Background(), srv.URL+"/container?sig=secret", "data.jsonl") + require.NoError(t, err) + assert.Equal(t, "rows", string(data)) + assert.Equal(t, "/container/data.jsonl", gotPath) + assert.Equal(t, "secret", gotSig) +} + +// A storage failure has to name the blob, since the container holds several +// and the status alone does not say which one was refused. +func TestDownloadBlobReportsTheStatusAndName(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), srv.URL+"/c", "data.jsonl") + require.Error(t, err) + assert.Contains(t, err.Error(), "403") + assert.Contains(t, err.Error(), "data.jsonl") +} + +// A malformed URI is the caller's mistake, and it is worth catching before a +// request goes out against a half-parsed address. +func TestBlobOperationsRejectAnUnparseableURI(t *testing.T) { + client := NewDatasetClientFromPipeline( + "https://example", runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), "://nope", "x.jsonl") + require.Error(t, err) + + _, err = client.ListContainerBlobs(context.Background(), "://nope") + require.Error(t, err) + + err = client.UploadBlob(context.Background(), "://nope", "x.jsonl", []byte("{}")) + require.Error(t, err) +} + +// Storage answers a listing in XML, and a shape that does not parse yields no +// names rather than a panic. +func TestParseBlobNames(t *testing.T) { + assert.Equal(t, []string{"a.jsonl", "b.json"}, + parseBlobNames(blobListing("a.jsonl", "b.json"))) + assert.Empty(t, parseBlobNames(blobListing())) + assert.Empty(t, parseBlobNames("not xml at all"), + "an unreadable listing is an empty one, not a crash") + assert.Empty(t, parseBlobNames( + ``), + "a nameless blob cannot be downloaded, so it is not offered") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go new file mode 100644 index 00000000000..0b8e11661ad --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// One .jsonl per dataset in one folder is the ordinary layout under ./evals. +// Scanning the directory instead of reading the declared file registers the +// rows of whichever sorts first under the other one's name, while the +// reconciler records the fingerprint of the declared file — so the two agree +// forever and the eval scores data nobody chose. +func TestReadFirstJSONLFile_ReadsTheNamedFileNotItsNeighbour(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "zebra.jsonl") + require.NoError(t, os.WriteFile(named, []byte("{\"pick\":\"me\"}\n"), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "alpha.jsonl"), []byte("{\"pick\":\"not me\"}\n"), 0o600)) + + content, err := ReadFirstJSONLFile(named) + require.NoError(t, err) + assert.Contains(t, content, `"me"`) + assert.NotContains(t, content, "not me") + + content, err = ReadFirstJSONLFile(dir) + require.NoError(t, err) + assert.Contains(t, content, "not me", "the directory form takes the first .jsonl") +} + +// The dataset extension strips the BOM before upload; this path uploads the +// same rows under `azd up` and has to agree, or the same file registers +// differently depending on which command sent it. +func TestReadFirstJSONLFile_StripsTheByteOrderMark(t *testing.T) { + dir := t.TempDir() + body := append([]byte{0xEF, 0xBB, 0xBF}, []byte("{\"query\":\"q\"}\n")...) + + named := filepath.Join(dir, "d.jsonl") + require.NoError(t, os.WriteFile(named, body, 0o600)) + + for _, path := range []string{named, dir} { + content, err := ReadFirstJSONLFile(path) + require.NoError(t, err) + assert.Truef(t, strings.HasPrefix(content, `{"query"`), + "the first row has to start with its own first key, got %q", content) + } +} + +// A file holding only a BOM has no rows, and registering it succeeds — the +// failure would surface at the run that scores it instead. +func TestReadFirstJSONLFile_RefusesAnEmptyFile(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "d.jsonl") + require.NoError(t, os.WriteFile(named, []byte{0xEF, 0xBB, 0xBF}, 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} + +// The reconciler validates rows before it publishes, but `dataset create` and +// `dataset update` do not go through it. Upload does not parse rows either, so +// without this a malformed line registers a version that looks healthy and only +// fails in the run that reads it. +func TestReadFirstJSONLFileRefusesAMalformedRow(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "rows.jsonl") + body := "{\"query\":\"ok\"}\n{not json}\n" + require.NoError(t, os.WriteFile(named, []byte(body), 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) + assert.Contains(t, err.Error(), "2", "the error has to name the line that is wrong") +} + +func TestReadFirstJSONLFileRefusesAnEmptyRow(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "rows.jsonl") + require.NoError(t, os.WriteFile(named, []byte("{\"query\":\"ok\"}\n{}\n"), 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go new file mode 100644 index 00000000000..ab2b389b1d9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// DatasetList is the paged response returned when listing datasets or the +// versions of one dataset. +type DatasetList struct { + Value []Dataset `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListDatasets returns the datasets registered on the project. +func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { + first, err := doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) + if err != nil { + return nil, err + } + return c.followPages(ctx, first) +} + +// ListDatasetVersions returns every version of a single dataset. +func (c *DatasetClient) ListDatasetVersions( + ctx context.Context, + name string, + apiVersion string, +) (*DatasetList, error) { + path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) + first, err := doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) + if err != nil { + return nil, err + } + return c.followPages(ctx, first) +} + +// DeleteDatasetVersion removes a single dataset version. +func (c *DatasetClient) DeleteDatasetVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// VersionOrder returns a sortable value for a version string, matching the +// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions +// sort lowest. +func VersionOrder(version string) float64 { + v := strings.TrimSpace(version) + if v == "" { + return -1 + } + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + // Fall back to trailing digits, e.g. "v3" -> 3. + i := len(v) + for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { + i-- + } + if i == len(v) { + return -1 + } + if n, err := strconv.Atoi(v[i:]); err == nil { + return float64(n) + } + return -1 +} + +// VersionGreater reports whether a is a strictly newer version than b. +// +// Both must be orderable; when either is not, the answer is false so an +// unparseable version never triggers a drift failure on its own. +func VersionGreater(a, b string) bool { + orderA, orderB := VersionOrder(a), VersionOrder(b) + if orderA < 0 || orderB < 0 { + return false + } + return orderA > orderB +} + +// LatestVersion returns the highest version in the list, falling back to the +// last entry when none of the versions can be ordered. +func LatestVersion(datasets []Dataset) string { + best := "" + // VersionOrder returns -1 for anything it cannot order, so the sentinel has + // to be -1 rather than lower: below it, the first version it cannot order + // becomes the running best and the fallback below never runs. + bestOrder := -1.0 + for _, d := range datasets { + if o := VersionOrder(d.Version); o > bestOrder { + bestOrder, best = o, d.Version + } + } + if best == "" && len(datasets) > 0 { + return datasets[len(datasets)-1].Version + } + return best +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go new file mode 100644 index 00000000000..947477266d7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bufio" + "bytes" + "encoding/json" + "math" + "os" + "path/filepath" + "strconv" + "strings" + + "azureaieval/internal/messages" +) + +// CreateDatasetRequest is the request body for creating (uploading) a dataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +// +// The field spelling is not consistent across the surface: the live +// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other +// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings +// are accepted here because binding only one silently yields an empty URI, +// which then fails much later at download time. +type Dataset struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + // camelCase spellings (project endpoint). + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + + // snake_case spellings. + BlobURI string `json:"blob_uri,omitempty"` + DataURI string `json:"data_uri,omitempty"` + ContentURI string `json:"content_uri,omitempty"` +} + +// ResolvedBlobURI returns the first URI the service supplied, across both +// spellings. An empty result means the dataset carries no downloadable URI and +// the caller must fetch a credential instead. +func (d *Dataset) ResolvedBlobURI() string { + for _, candidate := range []string{ + d.BlobURI, d.BlobURICamel, + d.DataURI, d.DataURICamel, + d.ContentURI, d.ContentURICamel, + } { + if candidate != "" { + return candidate + } + } + return "" +} + +// DatasetCredential is the response for dataset credential (SAS token) requests. +// The API returns a nested structure with blobReference and blobReferenceForConsumption. +type DatasetCredential struct { + // Flat fields (legacy format). + BlobURI string `json:"blob_uri,omitempty"` + SAS string `json:"sas,omitempty"` + SASUri string `json:"sas_uri,omitempty"` + + // Nested fields (current API format). + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` +} + +// BlobReference represents a blob storage reference with credentials. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds SAS credential details for blob access. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` + SASPath string `json:"sas,omitempty"` +} + +// ResolvedDownloadURI returns the URL to download the dataset. +// Prefers blobReferenceForConsumption.credential.sasUri (current API), +// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. +func (c *DatasetCredential) ResolvedDownloadURI() string { + // Current API format: nested blob references. + if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { + if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { + return uri + } + } + if c.BlobReference != nil && c.BlobReference.Credential != nil { + if uri := c.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + // Legacy flat format. + if c.SASUri != "" { + return c.SASUri + } + if c.BlobURI != "" && c.SAS != "" { + return c.BlobURI + "?" + c.SAS + } + return c.BlobURI +} + +// PendingUploadResponse is returned by the startPendingUpload endpoint. +// It contains a SAS URI for uploading blob data and the blob container URI. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID *string `json:"pendingUploadId,omitempty"` + PendingUploadType string `json:"pendingUploadType,omitempty"` + Version string `json:"version,omitempty"` +} + +// ResolvedUploadURI returns the SAS URI for uploading blobs. +func (p *PendingUploadResponse) ResolvedUploadURI() string { + if p.BlobReference != nil && p.BlobReference.Credential != nil { + if uri := p.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + return "" +} + +// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. +func (p *PendingUploadResponse) ResolvedBlobURI() string { + if p.BlobReference != nil { + return p.BlobReference.BlobURI + } + return "" +} + +// FinalizeDatasetRequest is the request body for finalizing a dataset version +// after blob upload. +type FinalizeDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Type string `json:"type"` + IsReference bool `json:"isReference"` + DataURI string `json:"dataUri"` +} + +// NextVersion computes the next dataset version string. +// +// Rules: +// 1. Empty → "1.0" +// 2. Parsable as a decimal number → increment by 1, format as "N.0" +// 3. Ends with trailing digits → increment the trailing numeric part +// 4. Otherwise → append ".1" +func NextVersion(current string) string { + current = strings.TrimSpace(current) + if current == "" { + return "1.0" + } + + // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). + if f, err := strconv.ParseFloat(current, 64); err == nil { + return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) + } + + // Find trailing digits and increment them. + i := len(current) - 1 + for i >= 0 && current[i] >= '0' && current[i] <= '9' { + i-- + } + if i < len(current)-1 { + prefix := current[:i+1] + n, err := strconv.Atoi(current[i+1:]) + if err == nil { + return prefix + strconv.Itoa(n+1) + } + } + + return current + ".1" +} + +// ReadFirstJSONLFile reads the rows to upload from a .jsonl file, or from the +// first .jsonl in a directory. +// +// A file path is read as itself. Resolving it to its directory and scanning +// would upload whichever .jsonl sorts first, so a project with one file per +// dataset would register the wrong rows under a name while recording the +// fingerprint of the declared file — the two would then agree forever. +// +// An empty file is refused here rather than uploaded: registering it succeeds, +// and the failure then surfaces at the run that tries to score it, which is a +// long way from the command that caused it. +func ReadFirstJSONLFile(path string) (string, error) { + if info, err := os.Stat(path); err == nil && !info.IsDir() { + data, err := os.ReadFile(path) //nolint:gosec // local artifact path + if err != nil { + return "", messages.ReadingPath(path, err) + } + return jsonlContent(filepath.Base(path), data) + } + + dir := path + entries, err := os.ReadDir(dir) + if err != nil { + return "", messages.ReadingDatasetDirectory(err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + if strings.EqualFold(filepath.Ext(e.Name()), ".jsonl") { + data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path + if err != nil { + return "", messages.ReadingPath(e.Name(), err) + } + return jsonlContent(e.Name(), data) + } + } + return "", messages.NoJSONLInDirectory(dir) +} + +// utf8BOM is what Windows editors and PowerShell's Set-Content write ahead of +// otherwise valid UTF-8. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// jsonlContent prepares one file's bytes for upload. +func jsonlContent(name string, data []byte) (string, error) { + // Uploaded as-is a BOM becomes part of the first row's first key, so every + // consumer of the dataset sees one malformed record. + data = bytes.TrimPrefix(data, utf8BOM) + if strings.TrimSpace(string(data)) == "" { + return "", messages.DatasetFileHasNoRows(name) + } + if err := validateJSONLRows(name, data); err != nil { + return "", err + } + return string(data), nil +} + +// validateJSONLRows refuses a file the service would happily store. +// +// Upload does not parse the rows, so one malformed line registers a version +// that looks healthy and only fails in the run that reads it. The reconciler +// checks this too, but `dataset create` and `dataset update` do not go through +// it, so the check belongs on the path every upload shares. +func validateJSONLRows(name string, data []byte) error { + scanner := bufio.NewScanner(bytes.NewReader(data)) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return messages.JSONLRowInvalid(name, line, err) + } + if len(row) == 0 { + return messages.JSONLRowEmpty(name, line) + } + } + return scanner.Err() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go new file mode 100644 index 00000000000..4e287c5ec21 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -0,0 +1,714 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefix for dataset endpoints. +const pathDatasets = "/datasets" + +// DatasetClient provides methods for dataset upload, download, and metadata retrieval. +type DatasetClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewDatasetClient creates a new DatasetClient. +func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-datasets", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataset registers a dataset with inline content (upload). +func (c *DatasetClient) CreateDataset( + ctx context.Context, + request *CreateDatasetRequest, + apiVersion string, +) (*Dataset, error) { + return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) +} + +// UploadNextVersion registers the next version of a dataset, discovering the +// current one from the service when currentVersion is empty. +// +// Prefer this over UploadNewVersion. That function derives the next version +// from whatever it is handed, so an empty value restarts at 1.0 and the +// service rejects the pending upload with a 409 +// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers +// almost always mean "the version after whatever is registered", which is what +// this does. +// +// The version listing is eventually consistent -- it returns nothing for a +// second or two after a version is created -- so an empty listing cannot be +// trusted to mean the dataset is new. A conflict is therefore treated as a +// stale read: the listing is re-read, and when it is still behind, the version +// just refused is taken as proof that it exists and the next one is tried. +// Trusting the listing alone left a second upload issued moments after the +// first reporting a 409 to the user for a publish that should simply have +// added a version. +func (c *DatasetClient) UploadNextVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + if currentVersion == "" { + latest, err := c.latestRegisteredVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + currentVersion = latest + } + + var err error + for range versionConflictAttempts { + var ds *Dataset + ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !IsVersionConflict(err) { + return ds, err + } + + // The version derived from currentVersion is taken, so it exists + // whatever the listing says. Prefer the listing when it has caught up + // and moved further ahead; otherwise step past what was just refused. + refused := NextVersion(currentVersion) + currentVersion = refused + // A listing failure is not fatal here: the refused version is already a + // correct next step, so only a listing that has moved further ahead + // changes the outcome. + latest, listErr := c.latestRegisteredVersion(ctx, name, apiVersion) + if listErr == nil && versionAtLeast(latest, refused) { + currentVersion = latest + } + } + return nil, err +} + +// versionConflictAttempts bounds the walk past versions the listing has not +// caught up with. Each attempt is one refused pending upload, so this is short. +const versionConflictAttempts = 4 + +// versionAtLeast reports whether a is a version at or beyond b. +func versionAtLeast(a, b string) bool { + if a == "" { + return false + } + return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a +} + +// latestRegisteredVersion returns the newest registered version. A dataset the +// service does not know, and a listing that has not caught up, both report an +// empty version and no error. Every other failure is returned: treating a 403 +// or a timeout as "no versions" would restart an existing dataset at 1.0. +func (c *DatasetClient) latestRegisteredVersion( + ctx context.Context, + name string, + apiVersion string, +) (string, error) { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err != nil { + if IsNotFound(err) { + return "", nil + } + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", nil + } + return LatestVersion(list.Value), nil +} + +// isVersionConflict reports whether the service refused the upload because the +// target version already exists. +func IsVersionConflict(err error) bool { + respErr, ok := errors.AsType[*azcore.ResponseError](err) + if !ok { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404, which is how it says a +// dataset does not exist yet. +// +// A failure part-way through a page walk is refused before the status is read: +// the first page answered, so the dataset exists, and reading that 404 as +// absence restarts an existing dataset at 1.0. +func IsNotFound(err error) bool { + if _, walking := errors.AsType[pageWalkError](err); walking { + return false + } + respErr, ok := errors.AsType[*azcore.ResponseError](err) + if !ok { + return false + } + return respErr.StatusCode == http.StatusNotFound +} + +// UploadNewVersion reads the first JSONL file from localDir, computes the next +// version from currentVersion, and uploads it as a new dataset version using +// the 3-step pending upload flow: +// 1. startPendingUpload -> get SAS URI +// 2. Upload blob to SAS URI +// 3. Finalize dataset version with dataUri +func (c *DatasetClient) UploadNewVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) +} + +// UploadVersion publishes the dataset at exactly this version. +// +// Separate from UploadNewVersion because its parameter is the version to +// count from, not the one to write: passing "1.0" there publishes 2.0. An +// author who declares a version means that version. +func (c *DatasetClient) UploadVersion( + ctx context.Context, + name string, + version string, + localDir string, + apiVersion string, +) (*Dataset, error) { + content, err := ReadFirstJSONLFile(localDir) + if err != nil { + return nil, messages.ReadingDatasetFromDir(localDir, err) + } + + newVersion := version + + // Step 1: Start pending upload to get a SAS URI. + pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) + if err != nil { + return nil, messages.StartingPendingUpload(err) + } + + uploadURI := pending.ResolvedUploadURI() + if uploadURI == "" { + return nil, messages.NoUploadURI() + } + + // Step 2: Upload the JSONL file to blob storage. + // One blob per dataset, which is what the container-listing fallback in + // DownloadDatasetContent expects to find. Naming it for the content instead + // would stop two racing publishes of one version overwriting each other -- + // but it leaves several .jsonl beside each other, and that fallback picks + // the first by name, so a download could return rows no version points at. + // The overwrite is the narrower harm and stays until the version can be + // allocated by the service rather than guessed from a lagging listing. + blobName := name + ".jsonl" + if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { + return nil, messages.UploadingBlob(err) + } + + // Step 3: Finalize the dataset version with the full blob URI. + dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) +} + +// StartPendingUpload initiates a pending upload for a dataset version. +// Returns the SAS URI and blob reference for uploading data. +func (c *DatasetClient) StartPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// blobHTTPClient is the client used for direct blob calls. +// +// Bounded: these bypass the SDK pipeline, so nothing else stops a hung storage +// endpoint from holding the command open until someone kills it. Generous, so +// a large dataset over a slow link still finishes. One client, so connections +// are reused across upload, finalize, list and download. +var blobHTTPClient = &http.Client{Timeout: 10 * time.Minute} + +// UploadBlob uploads data to a container SAS URI as a block blob. +func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return messages.InvalidContainerURI(urlsafe.Error(err)) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return messages.CreatingUploadRequest(urlsafe.Error(err)) + } + req.Header.Set("x-ms-blob-type", "BlockBlob") + req.Header.Set("Content-Type", "application/octet-stream") + + httpClient := blobHTTPClient + resp, err := httpClient.Do(req) + if err != nil { + return messages.UploadingBlobFailed(urlsafe.Error(err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return messages.BlobUploadStatus(resp.StatusCode, string(body)) + } + + return nil +} + +// FinalizeDatasetVersion completes the dataset version after blob upload +// by sending the metadata (name, version, dataUri) to the API. +func (c *DatasetClient) FinalizeDatasetVersion( + ctx context.Context, + name string, + version string, + dataURI string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + request := &FinalizeDatasetRequest{ + Name: name, + Version: version, + Type: "uri_file", + DataURI: dataURI, + } + return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) +} + +// GetDataset retrieves metadata for a dataset by name and version. +func (c *DatasetClient) GetDataset( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. +func (c *DatasetClient) GetDatasetCredential( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*DatasetCredential, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/credentials", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) +} + +// DownloadDatasetContent fetches a dataset version's content, whether its URI +// names a blob or a container. +// +// The two differ by origin, not by any field: a dataset uploaded through +// startPendingUpload gets a URI ending in the file name, while one produced by +// a generation job gets the container it was written into, with isSingleFile +// true either way. Downloading the container directly returns a 409, so the +// blob inside has to be found first. +// +// A credential is always fetched, because the URI on the dataset carries no +// SAS token and an unauthenticated read fails. +func (c *DatasetClient) DownloadDatasetContent( + ctx context.Context, + name string, + version string, + apiVersion string, +) ([]byte, error) { + cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) + if err != nil { + return nil, messages.ReadingDownloadCredentials(name, err) + } + + sasURI := cred.ResolvedDownloadURI() + if sasURI == "" { + return nil, messages.NoDownloadURI(name) + } + + // A URI whose last path segment carries a file extension is the blob + // itself; anything else is the container holding it. + if looksLikeBlobURI(sasURI) { + data, err := c.DownloadDataset(ctx, sasURI) + if err == nil { + return data, nil + } + log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) + } + + names, err := c.ListContainerBlobs(ctx, sasURI) + if err != nil { + return nil, messages.ListingDatasetContent(name, err) + } + blobName := pickDatasetBlob(names) + if blobName == "" { + return nil, messages.DatasetHasNoFile(name) + } + return c.DownloadBlob(ctx, sasURI, blobName) +} + +// looksLikeBlobURI reports whether the URI's final segment names a file. +func looksLikeBlobURI(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + last := path.Base(strings.TrimSuffix(u.Path, "/")) + return path.Ext(last) != "" +} + +// pickDatasetBlob chooses the file to read from a container, preferring JSONL +// since that is what an evaluation dataset is. +func pickDatasetBlob(names []string) string { + for _, n := range names { + if strings.EqualFold(path.Ext(n), ".jsonl") { + return n + } + } + for _, n := range names { + if n != "" && !strings.HasSuffix(n, "/") { + return n + } + } + return "" +} + +// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. +// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token +// (e.g., from DatasetCredential.ResolvedDownloadURI()). +func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { + req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) + if err != nil { + return nil, messages.CreatingDownloadRequest(urlsafe.Error(err)) + } + + // Use a plain HTTP client for blob downloads -- the SAS token in the URL provides + // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) + // should not be sent to Azure Blob Storage endpoints. + httpClient := blobHTTPClient + resp, err := httpClient.Do(req.Raw()) + if err != nil { + return nil, messages.DownloadingDatasetBlob(urlsafe.Error(err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, messages.BlobDownloadStatus(resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingDatasetContent(err) + } + + log.Printf("[dataset_api] downloaded %d bytes", len(data)) + return data, nil +} + +// ListContainerBlobs lists blobs in a container using a container-level SAS URI. +// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). +// Returns a list of blob names found in the container. +func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { + // Parse the container URI and append list query parameters. + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, messages.InvalidContainerURI(urlsafe.Error(err)) + } + + // The Blob service answers one page and a NextMarker. Only the marker value + // comes from the service -- the URL is the one built here -- so this walk + // carries none of the risk that following a body-supplied link would. + var names []string + marker := "" + for range maxListPages { + page := *u + q := page.Query() + q.Set("restype", "container") // cspell:ignore restype -- Azure Storage API query parameter + q.Set("comp", "list") + if marker != "" { + q.Set("marker", marker) + } + page.RawQuery = q.Encode() + + log.Printf("[dataset_api] listing blobs: %s", urlsafe.URL(&page)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, page.String(), nil) + if err != nil { + return nil, messages.CreatingListRequest(urlsafe.Error(err)) + } + + pageNames, next, err := c.readBlobPage(req) + if err != nil { + return nil, err + } + names = append(names, pageNames...) + if next == "" || next == marker { + break + } + marker = next + } + + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// readBlobPage performs one container listing request. +func (c *DatasetClient) readBlobPage(req *http.Request) ([]string, string, error) { + //nolint:gosec // the URI is the SAS the dataset service issued for this dataset, not caller input + resp, err := blobHTTPClient.Do(req) + if err != nil { + return nil, "", messages.ListingContainerBlobs(urlsafe.Error(err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", messages.ContainerListStatus(resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, "", messages.ReadingListResponse(err) + } + names, next := parseBlobPage(string(body)) + return names, next, nil +} + +// DownloadBlob downloads a single blob from a container using the container SAS URI +// and the blob name. Returns the blob content as bytes. +func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, messages.InvalidContainerURI(urlsafe.Error(err)) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, messages.CreatingBlobDownloadRequest(urlsafe.Error(err)) + } + + httpClient := blobHTTPClient + resp, err := httpClient.Do(req) + if err != nil { + return nil, messages.DownloadingBlob(urlsafe.Error(err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, messages.BlobDownloadStatusFor(resp.StatusCode, blobName) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingBlobContent(err) + } + + log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) + return data, nil +} + +// parseBlobNames extracts blob names from the Azure Blob Storage XML list response +// using proper XML parsing against the EnumerationResults schema. +func parseBlobNames(xmlBody string) []string { + names, _ := parseBlobPage(xmlBody) + return names +} + +// parseBlobPage extracts one page of blob names and the marker that continues +// the listing. An empty marker means this was the last page. +func parseBlobPage(xmlBody string) ([]string, string) { + type blob struct { + Name string `xml:"Name"` + } + type blobs struct { + Blob []blob `xml:"Blob"` + } + type enumerationResults struct { + Blobs blobs `xml:"Blobs"` + NextMarker string `xml:"NextMarker"` + } + + var result enumerationResults + if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { + return nil, "" + } + + names := make([]string, 0, len(result.Blobs.Blob)) + for _, b := range result.Blobs.Blob { + if b.Name != "" { + names = append(names, b.Name) + } + } + return names, result.NextMarker +} + +// doRequest performs an HTTP request against the dataset API and returns the raw response body. +func (c *DatasetClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + + // Callers escape the name and version they interpolate, so the path is set + // as the raw one. Assigning it to u.Path re-escapes the percent signs, and + // a dataset named "my dataset" then addresses one named "my%20dataset". + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, messages.InvalidRequestPath(escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + + log.Printf("[dataset_api] %s %s", method, urlsafe.URL(u)) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, messages.MarshalingRequest(err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, messages.SettingRequestBody(err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + + log.Printf("[dataset_api] response status: %d", resp.StatusCode) + + // 204 belongs here for the same reason it does in eval_api: a delete that + // removed the version answers No Content, and rejecting that reports every + // successful delete as an error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *DatasetClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, messages.ParsingResponse(err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go new file mode 100644 index 00000000000..00f4bb8ff19 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeCredential satisfies the constructor without reaching for a real token. +type fakeCredential struct{} + +func (fakeCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "fake", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +// recordedCall is one request the client made, as the service saw it. +type recordedCall struct { + method string + path string + rawPath string + apiVersion string +} + +// recordingDatasetClient answers every request with body and status, recording +// what was asked. Retries are off so a deliberate failure is one call. +func recordingDatasetClient(t *testing.T, status int, body string) (*DatasetClient, *[]recordedCall) { + t.Helper() + calls := &[]recordedCall{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *calls = append(*calls, recordedCall{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + apiVersion: r.URL.Query().Get("api-version"), + }) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline), calls +} + +// The paths are the service contract, and a wrong one costs a round trip to +// find out. Each is pinned against the shape the API documents. +func TestDatasetOperationPaths(t *testing.T) { + cases := []struct { + name string + call func(c *DatasetClient) error + wantMethod string + wantPath string + }{ + { + name: "list", + call: func(c *DatasetClient) error { _, err := c.ListDatasets(t.Context(), testAPIVersion); return err }, + wantMethod: http.MethodGet, + wantPath: "/datasets", + }, + { + name: "list versions", + call: func(c *DatasetClient) error { + _, err := c.ListDatasetVersions(t.Context(), "ds", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions", + }, + { + name: "get", + call: func(c *DatasetClient) error { + _, err := c.GetDataset(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "credential", + call: func(c *DatasetClient) error { + _, err := c.GetDatasetCredential(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/credentials", + }, + { + name: "start pending upload", + call: func(c *DatasetClient) error { + _, err := c.StartPendingUpload(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/startPendingUpload", + }, + { + name: "finalize", + call: func(c *DatasetClient) error { + _, err := c.FinalizeDatasetVersion(t.Context(), "ds", "1.0", "https://x/y.jsonl", testAPIVersion) + return err + }, + wantMethod: http.MethodPut, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "create", + call: func(c *DatasetClient) error { + _, err := c.CreateDataset(t.Context(), &CreateDatasetRequest{Name: "ds"}, testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets", + }, + { + name: "delete", + call: func(c *DatasetClient) error { return c.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion) }, + wantMethod: http.MethodDelete, + wantPath: "/datasets/ds/versions/1.0", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{"name":"ds","version":"1.0","value":[]}`) + require.NoError(t, tc.call(client)) + require.Len(t, *calls, 1) + assert.Equal(t, tc.wantMethod, (*calls)[0].method) + assert.Equal(t, tc.wantPath, (*calls)[0].path) + assert.Equal(t, testAPIVersion, (*calls)[0].apiVersion, + "the service rejects a request that names no api-version") + }) + } +} + +// A name is caller-supplied and a version can be anything the author wrote, so +// both are escaped rather than pasted into the path. +func TestDatasetPathsEscapeNameAndVersion(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{}`) + _, err := client.GetDataset(t.Context(), "my dataset/v", "1.0 beta", testAPIVersion) + require.NoError(t, err) + + require.Len(t, *calls, 1) + assert.Equal(t, "/datasets/my%20dataset%2Fv/versions/1.0%20beta", (*calls)[0].rawPath, + "an unescaped slash would address a different resource entirely") +} + +// A delete answers 204 with nothing in it, which must not read as a failure to +// parse a body that was never promised. +func TestDeleteDatasetVersionAcceptsNoContent(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusNoContent, "") + require.NoError(t, client.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion)) + assert.Len(t, *calls, 1) +} + +// The listing arrives wrapped in a value envelope; reading it flat yields an +// empty list rather than an error, which looks like a project with no datasets. +func TestListDatasetsReadsTheValueEnvelope(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusOK, + `{"value":[{"name":"a","version":"1.0"},{"name":"b","version":"2.0"}]}`) + + list, err := client.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err) + require.Len(t, list.Value, 2) + assert.Equal(t, "a", list.Value[0].Name) + assert.Equal(t, "2.0", list.Value[1].Version) +} + +// A failure has to surface as one, since the caller otherwise proceeds with a +// zero-valued dataset and fails somewhere further away. +func TestDatasetOperationsSurfaceServiceFailures(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetDataset(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + err = client.DeleteDatasetVersion(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + _, err = client.ListDatasets(t.Context(), testAPIVersion) + require.Error(t, err) +} + +// The constructor has to build a usable client — it wires the auth policies +// the live service needs, and nothing else exercises that path. +func TestNewDatasetClient(t *testing.T) { + client := NewDatasetClient("https://example.services.ai.azure.com/api/projects/p", fakeCredential{}) + require.NotNil(t, client) + assert.Equal(t, "https://example.services.ai.azure.com/api/projects/p", client.endpoint) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go new file mode 100644 index 00000000000..251214b7382 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Refusing IsNotFound must not cost the rest of the error's identity. +// +// The first version of this guard dropped Unwrap altogether, which did stop a +// later-page 404 reading as absence but also made a cancelled walk stop looking +// like a cancellation to everything upstream. +func TestAPageWalkFailureKeepsItsCause(t *testing.T) { + wrapped := pageWalkError{cause: context.Canceled} + + assert.True(t, errors.Is(wrapped, context.Canceled), + "a walk cancelled part-way through is still a cancellation") + assert.False(t, IsNotFound(wrapped), + "the first page answered, so the dataset is not missing") + assert.Contains(t, wrapped.Error(), "later page", + "and the message says which part of the listing failed") +} + +// A 404 on the first page means the service does not know this dataset. A 404 +// on a later page means the continuation failed -- the first page already +// proved the dataset exists. Reading the second as the first answered "no +// versions, no error", which restarts an existing dataset at 1.0. +func TestALaterPageFailingIsNotAbsence(t *testing.T) { + // The nextLink is built from the server's own URL rather than echoed back + // from the request: reflecting r.Host into a response body is a taint sink, + // and gosec is right to refuse it even in a test. + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "2" { + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte( + `{"value":[{"name":"ds","version":"3.0"}],"nextLink":"` + + srv.URL + `/datasets/ds/versions?page=2"}`)) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.ListDatasetVersions(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err) + assert.False(t, IsNotFound(err), + "the first page proved the dataset exists; a later 404 is the walk failing") + + version, err := client.latestRegisteredVersion(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err, "a failed walk must not answer with a version") + assert.Empty(t, version) + assert.Contains(t, strings.ToLower(err.Error()), "page") +} + +// The first page answering 404 is still absence, which is what lets a create +// know the name is free. +func TestAFirstPage404IsStillAbsence(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.ListDatasetVersions(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsNotFound(err)) + + version, err := client.latestRegisteredVersion(t.Context(), "ds", "2025-11-15-preview") + require.NoError(t, err, "an unknown dataset has no versions and that is not a failure") + assert.Empty(t, version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go new file mode 100644 index 00000000000..7027507f37a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +// maxListPages bounds page following so a service that keeps handing back a +// nextLink cannot spin forever. +const maxListPages = 100 + +// pageWalkError marks a failure that happened after the first page. +// +// The first page answered, so the dataset exists; a 404 on a later page is the +// continuation failing, not the dataset being unknown. IsNotFound refuses this +// wrapper for that reason -- but the cause is still reachable, so a cancelled +// context or an auth failure part-way through a walk classifies as itself +// rather than as an unreadable listing. +type pageWalkError struct{ cause error } + +func (e pageWalkError) Error() string { + return "reading a later page of the listing: " + e.cause.Error() +} + +func (e pageWalkError) Unwrap() error { return e.cause } + +// followPages walks nextLink until the service stops sending one, returning a +// single list holding every page. Without this, a project with more than one +// page lists incompletely and a latest-version check can decide from a stale +// first page. +func (c *DatasetClient) followPages(ctx context.Context, first *DatasetList) (*DatasetList, error) { + if first == nil { + return nil, nil + } + + // Copied rather than aliased: appending to first.Value could write into the + // caller's backing array when it has spare capacity. + out := &DatasetList{Value: append([]Dataset(nil), first.Value...)} + seen := map[string]bool{} + for next := first.NextLink; next != ""; { + if seen[next] || len(seen) >= maxListPages { + // A repeated or endless link is the service misbehaving, not a reason + // to fail the command -- but the list is short and, said through log, + // nobody would know: log goes to io.Discard unless --debug. + fmt.Fprint(os.Stderr, messages.Warning(messages.ListingTruncated(len(seen)))) + break + } + seen[next] = true + + body, err := c.doRequestGetURL(ctx, next) + if err != nil { + return nil, pageWalkError{cause: err} + } + var page DatasetList + // A page that answers 200 with no body ends the walk; unmarshaling it + // would throw away every page already collected. + if len(body) > 0 { + if err := json.Unmarshal(body, &page); err != nil { + return nil, messages.ParsingResponse(err) + } + } + out.Value = append(out.Value, page.Value...) + next = page.NextLink + } + return out, nil +} + +// sameOrigin reports whether two URLs share a scheme and host. +func sameOrigin(a, b *url.URL) bool { + return strings.EqualFold(a.Scheme, b.Scheme) && strings.EqualFold(a.Host, b.Host) +} + +// doRequestGetURL issues a GET against an absolute URL the service supplied, +// such as a nextLink. The URL is refused unless it shares the endpoint's +// origin: the pipeline attaches the caller's token, so a link pointing +// elsewhere would hand that token to another host. +func (c *DatasetClient) doRequestGetURL(ctx context.Context, rawURL string) ([]byte, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, messages.InvalidNextLink(rawURL, err) + } + base, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + + // A nextLink is allowed to be relative. Resolving it against the endpoint + // first keeps the origin check meaningful instead of rejecting a legitimate + // relative link for having no scheme or host of its own. + u := base.ResolveReference(parsed) + if !sameOrigin(u, base) { + return nil, messages.NextLinkOffOrigin(u.Scheme + "://" + u.Host) + } + + req, err := runtime.NewRequest(ctx, http.MethodGet, u.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + + log.Printf("[dataset_api] GET %s", urlsafe.URL(u)) + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + if !runtime.HasStatusCode(resp, http.StatusOK) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + return respBody, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go new file mode 100644 index 00000000000..cdf8949db04 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func datasetClientServing(t *testing.T, handler func(http.ResponseWriter, *http.Request, string)) *DatasetClient { + t.Helper() + var base string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r, base) + })) + t.Cleanup(srv.Close) + base = srv.URL + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline) +} + +// UploadVersion picks the next version from this listing, so a version sitting +// on page two meant reusing one that already exists. +func TestListDatasetVersionsFollowsNextLink(t *testing.T) { + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") == "2" { + fmt.Fprint(w, `{"value":[{"name":"golden","version":"3.0"}]}`) + return + } + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"},{"name":"golden","version":"2.0"}],`+ + `"nextLink":"%s/page?page=2"}`, base) + }) + + list, err := c.ListDatasetVersions(context.Background(), "golden", "v1") + + require.NoError(t, err) + require.Len(t, list.Value, 3, "both pages have to be gathered") + assert.Equal(t, "3.0", list.Value[2].Version, "the newest version was on page two") +} + +// The link arrives in a response body and this client sends an Authorization +// header, so following one to another host would send the token there. +func TestNextLinkToAnotherHostIsRefused(t *testing.T) { + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("the client followed a link off its own host: %s", r.URL) + })) + t.Cleanup(elsewhere.Close) + + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"}],"nextLink":"%s/steal"}`, + elsewhere.URL) + }) + + _, err := c.ListDatasets(context.Background(), "v1") + + require.Error(t, err) + assert.Contains(t, err.Error(), "refusing to follow", + "the refusal has to say the link was not followed") + assert.Contains(t, err.Error(), elsewhere.URL, + "and name the host it refused, so the reader can see what was in the body") +} + +// A link pointing at the page it came from is the one shape that would +// otherwise spin until the page bound for no benefit. +func TestSelfReferencingNextLinkStops(t *testing.T) { + calls := 0 + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"}],"nextLink":"%s/same"}`, base) + }) + + list, err := c.ListDatasets(context.Background(), "v1") + + require.NoError(t, err) + assert.Equal(t, 2, calls, "the first request, then the link once") + assert.Len(t, list.Value, 2) +} + +// A listing without a link is one page, which is what this did before it could +// see the link at all. +func TestListingWithoutANextLinkIsOnePage(t *testing.T) { + calls := 0 + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"value":[{"name":"golden","version":"1.0"}]}`) + }) + + list, err := c.ListDatasets(context.Background(), "v1") + + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Len(t, list.Value, 1) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go new file mode 100644 index 00000000000..390af92b3ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func pagingEdgeClient(t *testing.T, h http.HandlerFunc) (*DatasetClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)), srv +} + +// A nextLink is allowed to be relative, and a relative one has no host or +// scheme of its own. Comparing it to the endpoint before resolving refused a +// legitimate link and turned a working listing into a hard failure. +func TestListDatasetsFollowsARelativeNextLink(t *testing.T) { + c, _ := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"/datasets?page=2"}`) + return + } + fmt.Fprint(w, `{"value":[{"name":"two"}]}`) + }) + + list, err := c.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err, "a relative nextLink must be followed, not refused") + require.NotNil(t, list) + require.Len(t, list.Value, 2) + assert.Equal(t, "two", list.Value[1].Name) +} + +// A relative link still has to stay on the endpoint. Resolving must not become +// a way to reach another host by writing a protocol-relative link. +func TestListDatasetsRefusesAProtocolRelativeLinkToAnotherHost(t *testing.T) { + var elsewhereHits int32 + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&elsewhereHits, 1) + fmt.Fprint(w, `{"value":[{"name":"leaked"}]}`) + })) + t.Cleanup(elsewhere.Close) + + // "//host/path" resolves to the same scheme on a different host. + c, _ := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"value":[{"name":"one"}],"nextLink":"//%s/datasets"}`, + elsewhere.Listener.Addr().String()) + }) + + _, err := c.ListDatasets(t.Context(), testAPIVersion) + + require.Error(t, err, "a link resolving to another host must be refused") + assert.Zero(t, atomic.LoadInt32(&elsewhereHits), "the other host must never be contacted") +} + +// A cycle longer than one hop used to run to maxPages, because only a link +// pointing at the page it came from ended the walk. +func TestListDatasetsStopsOnATwoPageCycle(t *testing.T) { + var hits int32 + var base string + c, srv := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + // a -> b -> a, which never repeats the immediately previous link. + if r.URL.Query().Get("page") == "b" { + fmt.Fprintf(w, `{"value":[{"name":"b"}],"nextLink":%q}`, base+"/datasets?page=a") + return + } + fmt.Fprintf(w, `{"value":[{"name":"a"}],"nextLink":%q}`, base+"/datasets?page=b") + }) + base = srv.URL + + list, err := c.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err, "a cycle ends the walk rather than failing the command") + require.NotNil(t, list) + assert.LessOrEqual(t, atomic.LoadInt32(&hits), int32(4), + "a two-page cycle must stop quickly, not run to maxPages") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go new file mode 100644 index 00000000000..164cc6d415b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers the three-step publish, refusing any version in taken +// and reporting whatever the listing is told to report. +type uploadServer struct { + mu sync.Mutex + taken map[string]bool + listing []string + attempts []string +} + +func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + version := strings.Split(r.URL.Path, "/versions/")[1] + version = strings.TrimSuffix(version, "/startPendingUpload") + s.attempts = append(s.attempts, version) + if s.taken[version] { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReference": map[string]any{ + "blobUri": base() + "/c", + "storageAccountArmId": "id", + "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, + }, + })) + + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): + values := []map[string]any{} + for _, v := range s.listing { + values = append(values, map[string]any{"name": "ds", "version": v}) + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + + case r.Method == http.MethodPut: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + s.taken[version] = true + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "ds", "version": version, + })) + + default: + // The blob PUT. + w.WriteHeader(http.StatusCreated) + } + } +} + +// The version listing lags a publish, so a second upload can be told the +// dataset is new and restart at a version that already exists. Trusting the +// listing alone surfaced that 409 to the user for a publish that should simply +// have added a version. +func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"1.0": true}} + // The listing has not caught up: it still reports nothing at all. + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err, "a stale listing must not surface as a conflict") + assert.Equal(t, "2.0", ds.Version) + assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, + "the version just refused is proof it exists, so the next one is tried") +} + +// When the listing has caught up and is further ahead than the refused +// version, it is the better answer: it skips versions somebody else published. +func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { + server := &uploadServer{ + taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, + listing: []string{"1.0", "2.0", "3.0"}, + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "4.0", ds.Version) +} + +// A service that refuses everything must end in the conflict rather than +// looping: an unbounded walk would hammer the service on a real failure. +func TestUploadNextVersionGivesUpBounded(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}} + for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { + server.taken[v] = true + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsVersionConflict(err)) + assert.Len(t, server.attempts, versionConflictAttempts) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go new file mode 100644 index 00000000000..e6cc46c2e3d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service spells these fields inconsistently, and a URI read from the +// wrong spelling comes back empty rather than wrong — which is how the dataset +// URI went unbound the first time. +func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { + cases := map[string]string{ + `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", + `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", + } + for body, want := range cases { + var ds Dataset + require.NoError(t, json.Unmarshal([]byte(body), &ds), body) + assert.Equal(t, want, ds.ResolvedBlobURI(), body) + } + + var none Dataset + require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) + assert.Empty(t, none.ResolvedBlobURI(), + "no URI means the caller has to fetch a credential, not that the dataset is unreadable") +} + +// An upload needs the SAS-bearing URI to write to and the plain one to +// finalize with. Confusing them fails at different stages, so both are read +// from their own place. +func TestPendingUploadURIs(t *testing.T) { + var p PendingUploadResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReference": { + "blobUri": "https://acct.blob.core.windows.net/container", + "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } + } + }`), &p)) + + assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), + "the upload target carries the SAS") + assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), + "the finalize URI does not") + + var empty PendingUploadResponse + assert.Empty(t, empty.ResolvedUploadURI()) + assert.Empty(t, empty.ResolvedBlobURI()) +} + +// Credentials arrive in two shapes and the consumption one takes precedence, +// because that is the one scoped for reading. +func TestCredentialResolvedDownloadURI(t *testing.T) { + var c DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, + "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } + }`), &c)) + assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) + + var legacy DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) + assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), + "the flat spelling is still honoured") + + var none DatasetCredential + assert.Empty(t, none.ResolvedDownloadURI()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go new file mode 100644 index 00000000000..8db5d71f64e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func versionDatasetDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "data.jsonl"), []byte(`{"query":"hi"}`+"\n"), 0o600)) + return dir +} + +// An empty version listing is how a brand-new dataset looks, so a listing that +// failed must never be mistaken for one. Restarting at 1.0 against a dataset +// that already has versions either collides or publishes over the wrong one. +func TestUploadNextVersionRefusesToStartOverWhenTheListingFails(t *testing.T) { + var mu sync.Mutex + var paths []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + mu.Unlock() + + if strings.HasSuffix(r.URL.Path, "/versions") { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":{"code":"AuthorizationFailed"}}`)) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + c := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := c.UploadNextVersion(t.Context(), "ds", "", versionDatasetDir(t), testAPIVersion) + + require.Error(t, err, "a refused listing must surface, not read as a new dataset") + + mu.Lock() + defer mu.Unlock() + for _, p := range paths { + assert.NotContains(t, p, "startPendingUpload", + "no upload may be attempted once the version listing failed") + } +} + +// A 404 is the service saying the dataset does not exist, which genuinely means +// "no versions yet" and must stay distinguishable from a failure. +func TestUploadNextVersionTreatsAnUnknownDatasetAsVersionless(t *testing.T) { + var mu sync.Mutex + var startedVersion string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/versions"): + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"ResourceNotFound"}}`)) + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + mu.Lock() + v := strings.TrimSuffix(r.URL.Path, "/startPendingUpload") + startedVersion = v[strings.LastIndex(v, "/")+1:] + mu.Unlock() + w.WriteHeader(http.StatusInternalServerError) // stop here; the version is the point + default: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + c := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, _ = c.UploadNextVersion(t.Context(), "ds", "", versionDatasetDir(t), testAPIVersion) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, "1.0", startedVersion, "an unknown dataset still starts at 1.0") +} + +func TestIsNotFoundOnlyMatchesA404(t *testing.T) { + assert.False(t, IsNotFound(fmt.Errorf("plain error")), "a non-service error is not a 404") + assert.False(t, IsNotFound(nil), "no error is not a 404") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go new file mode 100644 index 00000000000..f4817100c51 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Drift detection compares the version on the service with the one recorded at +// the last deploy, so the ordering has to be numeric rather than lexical: +// "10.0" is newer than "9.0" even though it sorts earlier as a string. +func TestVersionGreater(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.0", "1.0", true}, + {"1.0", "2.0", false}, + {"1.0", "1.0", false}, + {"10.0", "9.0", true}, + {"9.0", "10.0", false}, + {"v3", "v2", true}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), + "VersionGreater(%q, %q)", tc.a, tc.b) + } +} + +// An unorderable version must never trigger a drift failure on its own: the +// deploy would be blocked with no way for the author to reason about it. +func TestVersionGreaterIgnoresUnorderable(t *testing.T) { + require.False(t, VersionGreater("draft", "1.0")) + require.False(t, VersionGreater("1.0", "draft")) + require.False(t, VersionGreater("", "1.0")) + require.False(t, VersionGreater("1.0", "")) +} + +// The two upload entry points read their version argument differently, and the +// difference is the whole point: UploadNewVersion counts from it, UploadVersion +// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what +// an author who wrote version: "1.0" asked for. +func TestNextVersionCountsFromTheArgument(t *testing.T) { + if got := NextVersion("1.0"); got != "2.0" { + t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) + } + if got := NextVersion("1"); got != "2.0" { + t.Fatalf("NextVersion(1) = %q, want 2.0", got) + } + // An unknown current version starts the sequence rather than guessing. + if got := NextVersion(""); got != "1.0" { + t.Fatalf("NextVersion(empty) = %q, want 1.0", got) + } +} + +func TestLatestVersionOrdersNumerically(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) + if got != "10.0" { + t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) + } + if LatestVersion(nil) != "" { + t.Fatal("LatestVersion(nil) should be empty") + } +} + +// LatestVersion documents a fallback to the last entry when nothing can be +// ordered. That fallback only runs if an unorderable version never becomes the +// running best, which a sentinel below -1 quietly prevented. +func TestLatestVersionFallsBackToTheLastEntryWhenNoneAreOrderable(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "alpha"}, {Version: "beta"}, {Version: "gamma"}}) + require.Equal(t, "gamma", got, "with nothing orderable the service's last entry wins") +} + +func TestLatestVersionPrefersAnOrderableVersionOverAnUnorderableOne(t *testing.T) { + require.Equal(t, "2.0", LatestVersion([]Dataset{{Version: "alpha"}, {Version: "2.0"}})) + require.Equal(t, "2.0", LatestVersion([]Dataset{{Version: "2.0"}, {Version: "alpha"}})) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go new file mode 100644 index 00000000000..f9ca13c3ad6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stuckCursorServer always answers has_more with the same last_id, which is +// what a service in a bad state does. Without a bound the client walks it +// forever, holding the command open and growing the slice until the process +// dies -- so this test would hang rather than fail if the guard were removed. +func stuckCursorServer(t *testing.T, calls *atomic.Int64) *EvalClient { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + body, err := json.Marshal(map[string]any{ + "data": []map[string]any{ + {"id": "item_1", "status": "pass"}, + }, + // The cursor never advances. + "has_more": true, + "last_id": "cursor_that_never_moves", + }) + assert.NoError(t, err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + return NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) +} + +// TestListOutputItemsStopsOnACursorThatNeverMoves pins termination. The +// deadline exists so a regression reports a failure instead of hanging the +// whole suite. +func TestListOutputItemsStopsOnACursorThatNeverMoves(t *testing.T) { + var calls atomic.Int64 + client := stuckCursorServer(t, &calls) + + done := make(chan struct{}) + var items *OutputItemList + var err error + go func() { + defer close(done) + items, err = client.ListOutputItems(t.Context(), "eval_1", "run_1", 0) + }() + + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("the walk never terminated on a repeating cursor") + } + + require.NoError(t, err) + require.NotNil(t, items) + assert.Equal(t, int64(2), calls.Load(), + "the repeat is visible on the second read, so the walk stops there") + assert.NotEmpty(t, items.Data, "the rows it did read are still returned") +} + +// A cursor that always advances defeats the repeat check, so the page ceiling +// is the only thing left holding the walk open. A service paging one row at a +// time forever would otherwise never return. +func TestListOutputItemsStopsAtThePageCeiling(t *testing.T) { + var calls atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := calls.Add(1) + body, err := json.Marshal(map[string]any{ + "data": []map[string]any{{"id": fmt.Sprintf("item_%d", n), "status": "pass"}}, + "has_more": true, + // Always a new cursor, so `seen` never fires. + "last_id": fmt.Sprintf("cursor_%d", n), + }) + assert.NoError(t, err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + done := make(chan struct{}) + var items *OutputItemList + var err error + go func() { + defer close(done) + items, err = client.ListOutputItems(t.Context(), "eval_1", "run_1", 0) + }() + + select { + case <-done: + case <-time.After(60 * time.Second): + t.Fatal("the walk never terminated on an endlessly advancing cursor") + } + + require.NoError(t, err) + require.NotNil(t, items) + assert.Equal(t, int64(maxPages), calls.Load(), + "the walk has to stop at the ceiling rather than trust the service to end it") + assert.Len(t, items.Data, maxPages) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go new file mode 100644 index 00000000000..3b4b93d7c7e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "errors" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsConflict reports whether the service refused because the resource is busy. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go new file mode 100644 index 00000000000..40df934ee14 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "log" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +// EvaluatorTypeBuiltin selects the platform-provided evaluators. +const EvaluatorTypeBuiltin = "Builtin" + +// JSONSchema is the subset of JSON Schema the evaluator contract uses. +type JSONSchema struct { + Type string `json:"type,omitempty"` + Required []string `json:"required,omitempty"` + Properties map[string]any `json:"properties,omitempty"` +} + +// PropertyNames returns the accepted property names, sorted for stable output. +func (s *JSONSchema) PropertyNames() []string { + if s == nil { + return nil + } + names := make([]string, 0, len(s.Properties)) + for name := range s.Properties { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Accepts reports whether the schema declares the named property. +func (s *JSONSchema) Accepts(name string) bool { + if s == nil || s.Properties == nil { + return false + } + _, ok := s.Properties[name] + return ok +} + +// EvaluatorContract is the published input contract for an evaluator: which +// data fields it consumes and which initialization parameters it takes. +type EvaluatorContract struct { + Type string `json:"type,omitempty"` + // PassThreshold is a pointer because an absent threshold and a zero one are + // different claims: zero passes every sample, absent defers to the + // `threshold:` init parameter on the criterion that uses this evaluator. + PassThreshold *float64 `json:"pass_threshold,omitempty"` + DataSchema *JSONSchema `json:"data_schema,omitempty"` + InitParameters *JSONSchema `json:"init_parameters,omitempty"` +} + +// EvaluatorSummary is a single entry in an evaluator listing. +// +// The listing carries the full contract, so callers can shape a request to +// match an evaluator instead of guessing and taking a service-side rejection. +type EvaluatorSummary struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Description string `json:"description,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + + // The listing spells this evaluator_type; `type` is accepted too because + // other evaluator payloads use it. + EvaluatorType string `json:"evaluator_type,omitempty"` + TypeAlias string `json:"type,omitempty"` + + Categories []string `json:"categories,omitempty"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` + Definition *EvaluatorContract `json:"definition,omitempty"` +} + +// Type reports the evaluator kind across both spellings. +func (e *EvaluatorSummary) Type() string { + if e.EvaluatorType != "" { + return e.EvaluatorType + } + return e.TypeAlias +} + +// SupportsLevel reports whether the evaluator runs at the given evaluation +// level. An evaluator that declares no levels is treated as unconstrained. +func (e *EvaluatorSummary) SupportsLevel(level string) bool { + if level == "" || len(e.SupportedEvaluationLevels) == 0 { + return true + } + for _, supported := range e.SupportedEvaluationLevels { + if strings.EqualFold(supported, level) { + return true + } + } + return false +} + +// DataSchema returns the evaluator's input schema, or nil when the listing +// did not describe one. +func (e *EvaluatorSummary) DataSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.DataSchema +} + +// InitSchema returns the evaluator's initialization-parameter schema, or nil +// when the listing did not describe one. +func (e *EvaluatorSummary) InitSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.InitParameters +} + +// EvaluatorListResponse is the paged response for an evaluator listing. +type EvaluatorListResponse struct { + Value []EvaluatorSummary `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ByName indexes the listing by evaluator name. +func (r *EvaluatorListResponse) ByName() map[string]*EvaluatorSummary { + if r == nil { + return nil + } + index := make(map[string]*EvaluatorSummary, len(r.Value)) + for i := range r.Value { + index[r.Value[i].Name] = &r.Value[i] + } + return index +} + +// ListEvaluators returns the evaluators visible to the project. Pass +// EvaluatorTypeBuiltin to list only the platform's built-ins. +func (c *EvalClient) ListEvaluators( + ctx context.Context, + evaluatorType string, + apiVersion string, +) (*EvaluatorListResponse, error) { + var query map[string]string + if evaluatorType != "" { + query = map[string]string{"type": evaluatorType} + } + first, err := doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, pathEvaluators, query, nil, apiVersion, + ) + if err != nil { + return nil, err + } + return walkNextLinks(ctx, c, first, + func(l *EvaluatorListResponse) string { return l.NextLink }, + func(into, page *EvaluatorListResponse) { into.Value = append(into.Value, page.Value...) }) +} + +// ListEvaluatorVersions returns every version of one evaluator. +func (c *EvalClient) ListEvaluatorVersions( + ctx context.Context, + name string, + apiVersion string, +) (*EvaluatorListResponse, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + first, err := doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, path, nil, nil, apiVersion, + ) + if err != nil { + return nil, err + } + return walkNextLinks(ctx, c, first, + func(l *EvaluatorListResponse) string { return l.NextLink }, + func(into, page *EvaluatorListResponse) { into.Value = append(into.Value, page.Value...) }) +} + +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + +// parseVersionNumber reads a version string as an integer, answering 0 for one +// that is not numeric. +func parseVersionNumber(version string) int { + number, err := strconv.Atoi(version) + if err != nil { + return 0 + } + return number +} + +// DeleteEvaluatorVersion removes a single evaluator version. +func (c *EvalClient) DeleteEvaluatorVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// CancelOpenAIEvalRun stops an in-flight run. +// +// The body must stay nil: this route cancels only when the body is empty, and +// updates the run's status and counters when it is not. +func (c *EvalClient) CancelOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") +} + +// DeleteOpenAIEvalRun removes a single run. +func (c *EvalClient) DeleteOpenAIEvalRun(ctx context.Context, evalID, runID string) error { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// ListOutputItems returns a run's per-sample results. +// +// The run itself carries only totals and a per-criterion breakdown. The output +// items are the rows: each one holds the dataset item that was evaluated, what +// the target answered, and every evaluator's score, verdict and reason. Showing +// results without them can say how many failed but never which, or why. +func (c *EvalClient) ListOutputItems( + ctx context.Context, + evalID, runID string, + limit int, +) (*OutputItemList, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + + // Pages are followed only when the service says there are more. A run of + // 200 samples answered one page at a time would otherwise be reported as + // however many rows fit in the first, and the mean scores computed from + // them would be a sample of the run rather than the run. + all := &OutputItemList{} + after := "" + // A cursor that repeats while still returning rows would spin forever and + // grow all.Data until the process dies, so the walk is bounded the same way + // the next-link walker in pages.go is. + seen := map[string]bool{} + for range maxPages { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit - len(all.Data)) + } + if after != "" { + query["after"] = after + } + + page, err := doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") + if err != nil { + return nil, err + } + all.Data = append(all.Data, page.Data...) + + if !page.HasMore || page.LastID == "" || len(page.Data) == 0 { + return all, nil + } + if limit > 0 && len(all.Data) >= limit { + return all, nil + } + if seen[page.LastID] { + log.Printf("[eval_api] cursor %q repeated; the listing may be incomplete", page.LastID) + return all, nil + } + seen[page.LastID] = true + after = page.LastID + } + log.Printf("[eval_api] stopped after %d pages; the listing may be incomplete", maxPages) + return all, nil +} + +// GetOutputItem reads a single evaluated row. +func (c *EvalClient) GetOutputItem( + ctx context.Context, + evalID, runID, itemID string, +) (*OutputItem, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), url.PathEscape(itemID), + ) + return doRequestTyped[OutputItem](c, ctx, http.MethodGet, path, nil, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go new file mode 100644 index 00000000000..8c84fb95e4e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The listing carries each evaluator's contract so a request can be shaped to +// match it. Reading that contract wrong means shaping the request wrong and +// taking a service-side rejection instead. +func TestEvaluatorSummaryReadsTheContract(t *testing.T) { + var summary EvaluatorSummary + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "relevance", + "version": "3", + "evaluator_type": "Builtin", + "supported_evaluation_levels": ["Run", "Turn"], + "definition": { + "data_schema": {"type":"object","required":["query"], + "properties":{"query":{},"response":{},"context":{}}}, + "init_parameters": {"type":"object","properties":{"model_config":{},"threshold":{}}} + } + }`), &summary)) + + assert.Equal(t, "Builtin", summary.Type()) + assert.Equal(t, []string{"context", "query", "response"}, summary.DataSchema().PropertyNames(), + "sorted, so a listing does not reorder itself between runs") + assert.True(t, summary.DataSchema().Accepts("response")) + assert.False(t, summary.DataSchema().Accepts("ground_truth")) + assert.Equal(t, []string{"model_config", "threshold"}, summary.InitSchema().PropertyNames()) +} + +// The kind arrives under two names depending on the route. Reading only one +// leaves the type blank, which then reads as a custom evaluator. +func TestEvaluatorTypeAcceptsEitherSpelling(t *testing.T) { + spelled := EvaluatorSummary{EvaluatorType: "Builtin"} + aliased := EvaluatorSummary{TypeAlias: "Builtin"} + + assert.Equal(t, "Builtin", spelled.Type()) + assert.Equal(t, "Builtin", aliased.Type()) + assert.Empty(t, (&EvaluatorSummary{}).Type()) +} + +// An evaluator that declares no levels runs at any of them. Treating an empty +// list as "supports nothing" would reject every evaluator the listing does not +// describe fully. +func TestSupportsLevel(t *testing.T) { + constrained := EvaluatorSummary{SupportedEvaluationLevels: []string{"Run", "Turn"}} + assert.True(t, constrained.SupportsLevel("Run")) + assert.True(t, constrained.SupportsLevel("run"), "the level is matched without regard to case") + assert.False(t, constrained.SupportsLevel("Conversation")) + assert.True(t, constrained.SupportsLevel(""), "asking about no level is not a constraint") + + unconstrained := EvaluatorSummary{} + assert.True(t, unconstrained.SupportsLevel("Conversation"), + "an evaluator that declares no levels is unconstrained, not unusable") +} + +// A missing definition has to read as "not described", not crash the caller +// that asked what an evaluator accepts. +func TestEvaluatorSchemasTolerateAnAbsentDefinition(t *testing.T) { + var absent *EvaluatorSummary + assert.Nil(t, absent.DataSchema()) + assert.Nil(t, absent.InitSchema()) + + bare := EvaluatorSummary{Name: "custom"} + assert.Nil(t, bare.DataSchema()) + assert.Nil(t, bare.InitSchema()) + + var noSchema *JSONSchema + assert.Nil(t, noSchema.PropertyNames()) + assert.False(t, noSchema.Accepts("query")) + assert.False(t, (&JSONSchema{}).Accepts("query")) +} + +// The index is how an eval.yaml entry is matched to what the project offers. +func TestByName(t *testing.T) { + list := &EvaluatorListResponse{Value: []EvaluatorSummary{ + {Name: "relevance", Version: "3"}, + {Name: "coherence", Version: "1"}, + }} + + index := list.ByName() + require.Len(t, index, 2) + assert.Equal(t, "3", index["relevance"].Version) + assert.Nil(t, index["missing"]) + + var absent *EvaluatorListResponse + assert.Nil(t, absent.ByName()) +} + +// Listing built-ins is a filter on the same route, and dropping the parameter +// returns the project's custom evaluators mixed in. +func TestListEvaluatorsFiltersByType(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"value":[{"name":"relevance"}]}`) + + _, err := client.ListEvaluators(context.Background(), EvaluatorTypeBuiltin, "v1") + require.NoError(t, err) + assert.Equal(t, "/evaluators", last.path) + assert.Equal(t, "Builtin", last.query.Get("type")) + + _, err = client.ListEvaluators(context.Background(), "", "v1") + require.NoError(t, err) + assert.Empty(t, last.query.Get("type"), "no filter asks for everything") +} + +// This route cancels only when the body is empty, and updates the run's status +// and counters when it is not. The generation-job cancel next door requires an +// empty object, so the two are easy to unify into a bug that silently rewrites +// a run instead of stopping it. +func TestCancelOpenAIEvalRunSendsNoBody(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"run_1","status":"canceled"}`) + + _, err := client.CancelOpenAIEvalRun(context.Background(), "eval_1", "run_1") + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/openai/v1/evals/eval_1/runs/run_1", last.path) + assert.Empty(t, last.body, "a body turns this cancel into an update") +} + +// The run routes are OpenAI-compatible and carry no api-version; sending one +// is answered by a different contract than the client parses. +func TestRunRoutesAndTheirPaths(t *testing.T) { + cases := []struct { + name string + call func(c *EvalClient) error + wantMethod string + wantPath string + }{ + { + name: "delete run", + call: func(c *EvalClient) error { return c.DeleteOpenAIEvalRun(context.Background(), "eval_1", "run_1") }, + wantMethod: http.MethodDelete, + wantPath: "/openai/v1/evals/eval_1/runs/run_1", + }, + { + name: "list output items", + call: func(c *EvalClient) error { + _, err := c.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/openai/v1/evals/eval_1/runs/run_1/output_items", + }, + { + name: "get output item", + call: func(c *EvalClient) error { + _, err := c.GetOutputItem(context.Background(), "eval_1", "run_1", "item_1") + return err + }, + wantMethod: http.MethodGet, + wantPath: "/openai/v1/evals/eval_1/runs/run_1/output_items/item_1", + }, + { + name: "delete evaluator version", + call: func(c *EvalClient) error { + return c.DeleteEvaluatorVersion(context.Background(), "custom", "2", "v1") + }, + wantMethod: http.MethodDelete, + wantPath: "/evaluators/custom/versions/2", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"data":[]}`) + require.NoError(t, tc.call(client)) + assert.Equal(t, tc.wantMethod, last.method) + assert.Equal(t, tc.wantPath, last.path) + }) + } +} + +// A limit is only sent when asked for: sending limit=0 asks the service for +// nothing rather than for everything. +func TestListOutputItemsSendsTheLimitOnlyWhenSet(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"data":[]}`) + + _, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 50) + require.NoError(t, err) + assert.Equal(t, "50", last.query.Get("limit")) + + _, err = client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + require.NoError(t, err) + assert.Empty(t, last.query.Get("limit"), "no limit means the service's default, not zero rows") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go new file mode 100644 index 00000000000..46f7524acbf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Evaluator versions are integers rendered as strings, so a lexical compare +// ranks "9" above "15". The live service already has evaluators at version 15 +// and 17, so this is not hypothetical. +func TestPickLatestEvaluatorVersionIsNumeric(t *testing.T) { + cases := []struct { + name string + versions []string + want string + }{ + {"single", []string{"1"}, "1"}, + {"ascending", []string{"1", "2", "3"}, "3"}, + {"unordered", []string{"3", "1", "2"}, "3"}, + {"double digits beat single", []string{"9", "15"}, "15"}, + {"realistic", []string{"1", "9", "10", "17", "2"}, "17"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries := make([]EvaluatorSummary, 0, len(tc.versions)) + for _, v := range tc.versions { + entries = append(entries, EvaluatorSummary{Name: "e", Version: v}) + } + require.Equal(t, tc.want, pickLatestVersion(entries)) + }) + } +} + +// A non-numeric version is only used when nothing numeric exists, so one odd +// entry cannot mask the real latest. +func TestPickLatestEvaluatorVersionHandlesNonNumeric(t *testing.T) { + require.Equal(t, "2", pickLatestVersion([]EvaluatorSummary{ + {Version: "draft"}, {Version: "1"}, {Version: "2"}, + })) + require.Equal(t, "draft", pickLatestVersion([]EvaluatorSummary{{Version: "draft"}})) + require.Equal(t, "", pickLatestVersion(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go new file mode 100644 index 00000000000..f0edca1ada0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// Generation source building +// --------------------------------------------------------------------------- + +// TraceOptions holds optional trace inclusion parameters for generation sources. +type TraceOptions struct { + Days int +} + +// WithoutAgentSource returns the sources with the agent entry removed. +// +// Agent-seeded data generation currently fails server-side for every agent, +// while the same request carrying only the prompt succeeds, so this is what a +// retry falls back to. +func WithoutAgentSource(sources []GenerationSource) []GenerationSource { + kept := make([]GenerationSource, 0, len(sources)) + for _, s := range sources { + if s.Type == "agent" { + continue + } + kept = append(kept, s) + } + return kept +} + +// HasPromptSource reports whether anything remains to generate from. +func HasPromptSource(sources []GenerationSource) bool { + for _, s := range sources { + if s.Type == "prompt" && s.Prompt != "" { + return true + } + } + return false +} + +// BuildGenerationSources emits the sources the caller selected, in a stable +// order, along with the ones it asked for and nothing could be built from. +// +// kinds is what --from named. An empty kinds means "whatever this plan has to +// offer" and reports nothing missing: the caller expressed no preference, so +// there is nothing to disappoint. Naming a kind explicitly is a request, and a +// request that cannot be built is worth saying out loud rather than quietly +// submitting a job seeded from less than was asked for. +func BuildGenerationSources( + kinds []string, + agentName, version, instruction string, + traces *TraceOptions, +) (sources []GenerationSource, unbuildable []string) { + want := map[string]bool{} + for _, k := range kinds { + want[k] = true + } + // Empty kinds selects everything available; a populated one selects only + // what it names. + selected := func(kind string) bool { + return len(want) == 0 || want[kind] + } + // asked distinguishes "the default swept this up" from "the user typed it", + // which is what decides whether an empty-handed source is an error. + asked := func(kind string) bool { return want[kind] } + + // A traces source names the agent whose traces to read, but the service + // still requires a prompt or an agent beside it, so traces on their own are + // refused for every agent. The agent travels with them for the same reason + // it travels with a prompt below. Verified against the service: traces alone + // is a 400, traces plus agent is accepted. + tracesNeedTheAgent := asked("traces") && agentName != "" + + // The agent is settled first because whether it was built decides whether + // its instructions have anything to be the instructions of. + var agentSource *GenerationSource + if selected("agent") || tracesNeedTheAgent { + switch { + case agentName != "": + agentSource = &GenerationSource{Type: "agent", AgentName: agentName} + if version != "" { + agentSource.AgentVersion = version + } + case asked("agent"): + unbuildable = append(unbuildable, "agent") + } + } + + // Generating from an agent means generating from its instructions, so they + // travel with it as a prompt. That is also the only shape the service + // currently honours: the agent source alone fails for every agent, and the + // prompt is what the retry in generateDataset falls back to. Without this, + // `--from agent` would be a request that always fails, and `--from traces` + // would have nothing to fall back to when agent seeding fails. + promptCarriesTheAgent := agentSource != nil && (asked("agent") || tracesNeedTheAgent) + if selected("prompt") || promptCarriesTheAgent { + switch { + case instruction != "": + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + case asked("prompt"): + unbuildable = append(unbuildable, "prompt") + } + } + + if agentSource != nil { + sources = append(sources, *agentSource) + } + + if selected("traces") { + // A window narrows the request; it does not authorize it. Asking for + // traces without one means every trace the agent has. + switch { + case agentName == "" && asked("traces"): + // Without an agent the source names nothing to read and carries + // nothing the service accepts beside it. + unbuildable = append(unbuildable, "traces") + case traces != nil && traces.Days > 0: + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), + }) + case asked("traces"): + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + }) + } + } + + // The service takes a file's rows through the dataset upload path, not + // through a generation source, so there is nothing here to build one from. + if asked("file") { + unbuildable = append(unbuildable, "file") + } + + return sources, unbuildable +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the +// provided parameters. Currently, it's always "simple_qna" type with multiple sources +func NewDataGenerationJobRequest( + name, evalModel string, + maxSamples int, + sources []GenerationSource, +) *DataGenerationJobRequest { + return &DataGenerationJobRequest{ + Inputs: DataGenerationInputs{ + Name: name, + Scenario: "evaluation", + Options: DataGenerationOptions{ + Type: "simple_qna", + MaxSamples: maxSamples, + ModelOptions: ModelOptions{ + Model: evalModel, + }, + }, + Sources: sources, + }, + } +} + +// NewEvaluatorGenerationJobRequest builds an EvaluatorGenerationJobRequest +// from the provided parameters. +func NewEvaluatorGenerationJobRequest( + name, evalModel string, + sources []GenerationSource, +) *EvaluatorGenerationJobRequest { + return &EvaluatorGenerationJobRequest{ + Inputs: EvaluatorGenerationInputs{ + Name: name, + EvaluatorName: name, + Model: evalModel, + Sources: sources, + }, + } +} + +// --------------------------------------------------------------------------- +// Evaluator classification +// --------------------------------------------------------------------------- + +// IsBuiltinEvaluator returns true when the evaluator name has the "builtin." +// prefix. +func IsBuiltinEvaluator(name string) bool { + return strings.HasPrefix(name, "builtin.") +} + +// SplitEvaluators partitions evaluators into generated (non-builtin) and +// built-in lists. +func SplitEvaluators(evaluators evalcore.EvaluatorList) (generated, builtin evalcore.EvaluatorList) { + for _, e := range evaluators { + // Name labels the criterion in results and is empty for a plain + // `- evaluator: builtin.coherence`, so testing it classified every + // built-in as generated. Evaluator is the reference IsBuiltin reads. + if e.IsBuiltin() { + builtin = append(builtin, e) + } else { + generated = append(generated, e) + } + } + return generated, builtin +} + +// --------------------------------------------------------------------------- +// Dataset name detection +// --------------------------------------------------------------------------- + +// IsDatasetName returns true when the value looks like a registered dataset +// name rather than a local file path. A name has no path separators and no +// common data-file extension (.jsonl, .json, .csv). +func IsDatasetName(value string) bool { + if value == "" { + return false + } + if strings.ContainsAny(value, "/\\") { + return false + } + ext := strings.ToLower(filepath.Ext(value)) + return ext != ".jsonl" && ext != ".json" && ext != ".csv" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go new file mode 100644 index 00000000000..c936ba71c87 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation-job listings answer with the same has_more/last_id cursor the +// OpenAI listings use, but GenerationJobList did not carry those fields, so +// both read one page and stopped. Against the shared bug bash project that meant +// `job list` reported the first twenty jobs of many, with nothing to say so. +func TestGenerationJobListingsFollowTheCursor(t *testing.T) { + for _, tc := range []struct { + name string + path string + list func(*EvalClient) (*GenerationJobList, error) + }{ + { + name: "dataset jobs", + path: "/data_generation_jobs", + list: func(c *EvalClient) (*GenerationJobList, error) { + return c.ListDataGenerationJobs(t.Context(), "2025-11-15-preview") + }, + }, + { + name: "evaluator jobs", + path: "/evaluator_generation_jobs", + list: func(c *EvalClient) (*GenerationJobList, error) { + return c.ListEvaluatorGenerationJobs(t.Context(), "2025-11-15-preview") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var afters []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, tc.path, r.URL.Path) + after := r.URL.Query().Get("after") + afters = append(afters, after) + + w.WriteHeader(http.StatusOK) + switch after { + case "": + _, _ = w.Write([]byte( + `{"data":[{"id":"j1"},{"id":"j2"}],"has_more":true,"last_id":"j2"}`)) + case "j2": + _, _ = w.Write([]byte( + `{"data":[{"id":"j3"}],"has_more":false,"last_id":"j3"}`)) + default: + t.Errorf("unexpected cursor %q", after) + } + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + list, err := tc.list(client) + require.NoError(t, err) + require.Len(t, list.Data, 3, "every page should be gathered, not just the first") + assert.Equal(t, []string{"", "j2"}, afters, + "the second page should be asked for with the cursor the first returned") + }) + } +} + +// A listing that fits in one page must not ask for a second. +func TestGenerationJobListingStopsWithoutACursor(t *testing.T) { + var requests int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"data":[{"id":"j1"}],"has_more":false,"last_id":"j1"}`)) + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + list, err := client.ListDataGenerationJobs(t.Context(), "2025-11-15-preview") + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, requests) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go new file mode 100644 index 00000000000..298347a8e19 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -0,0 +1,270 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// kindsOf reduces the built sources to what --from talks about, which is the +// only part these tests are asserting on. +func kindsOf(sources []GenerationSource) []string { + kinds := make([]string, 0, len(sources)) + for _, s := range sources { + kinds = append(kinds, s.Type) + } + return kinds +} + +// Naming a source is a request to send that one, not a hint. Everything the +// plan could otherwise have offered stays out of the request. +// +// Demonstrated with prompt because it stands alone. Agent and traces are both +// refused by the service unless a prompt or an agent accompanies them, so each +// carries one; that carve-out is pinned in their own tests. +func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"prompt"}, + "support-agent", "3", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Generating from an agent means generating from its instructions, so asking +// for the agent carries them. It is also the only shape the service honours: +// the agent source on its own fails for every agent, so a `--from agent` that +// dropped the prompt would be a request that always fails. +func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "support-agent", "3", "answer support questions", nil, + ) + + assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) + assert.Equal(t, "answer support questions", sources[0].Prompt) + assert.Empty(t, unbuildable) +} + +// The instructions ride along with the agent; they do not stand in for it. An +// agent nobody named is still nothing to generate from. +func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "", "", "answer support questions", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"agent"}, unbuildable) +} + +// The agent name travels with the traces source: it is what scopes the query +// to this agent's conversations rather than the whole project's. +// +// The agent and its instructions are also sent as sources of their own, because +// the service refuses a request carrying neither a prompt nor an agent. +// Verified against it: traces alone is a 400 naming that requirement, traces +// plus agent is accepted. The prompt is what the agent-seeding retry falls back +// to, so without it a traces run has no way through that failure. +func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.True(t, HasPromptSource(WithoutAgentSource(sources)), + "dropping the agent must leave something the service still accepts") +} + +// Without an agent the traces source names nothing to read, and nothing the +// service accepts can accompany it, so it is refused here rather than sent. +func TestBuildGenerationSources_TracesNeedAnAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "", "", "", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"traces"}, unbuildable) +} + +// A day window narrows the trace query; it is not what authorizes it. The +// documented `dataset generate --from traces` carries no window, and it +// has to mean "every trace" rather than "no traces". +func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", nil, + ) + + require.Len(t, sources, 2) + assert.Equal(t, "traces", sources[1].Type) + assert.Zero(t, sources[1].StartTime, + "an absent window must leave start_time off the wire, not pin it to now") + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 2) + want := time.Now().AddDate(0, 0, -7).Unix() + assert.InDelta(t, want, sources[1].StartTime, 60) +} + +// No --from is no preference, so the plan sends everything it happens to have. +func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Expressing no preference cannot disappoint one, so an empty --from reports +// nothing missing however little the plan turns out to hold. +func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { + sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) + + assert.Empty(t, sources) + assert.Empty(t, unbuildable) +} + +// Asking for a source the plan cannot build has to surface, because the job is +// billed and what comes back looks the same either way. +func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { + tests := []struct { + name string + kinds []string + agentName string + instruction string + want []string + }{ + { + name: "prompt without an instruction", + kinds: []string{"prompt"}, + want: []string{"prompt"}, + }, + { + name: "agent without a target", + kinds: []string{"agent"}, + want: []string{"agent"}, + }, + { + name: "file is not a generation source at all", + kinds: []string{"file"}, + want: []string{"file"}, + }, + { + name: "several at once", + kinds: []string{"prompt", "agent"}, + want: []string{"agent", "prompt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + tt.kinds, tt.agentName, "", tt.instruction, nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, tt.want, unbuildable) + }) + } +} + +// A request that names two sources and can only build one still reports the +// one it could not, rather than being satisfied by the other's success. +func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent", "prompt"}, "support-agent", "", "", nil, + ) + + assert.Equal(t, []string{"agent"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, unbuildable) +} + +// `file` is only unbuildable when it was asked for. The default sweep must not +// invent a complaint about a source nobody named. +func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { + _, unbuildable := BuildGenerationSources( + nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, + ) + + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { + withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) + require.Len(t, withVersion, 1) + assert.Equal(t, "3", withVersion[0].AgentVersion) + + withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) + require.Len(t, withoutVersion, 1) + assert.Empty(t, withoutVersion[0].AgentVersion) +} + +// The retry that saves the documented flow: agent-seeded generation fails +// server-side for every agent, and the same request without the agent source +// succeeds. +func TestWithoutAgentSource(t *testing.T) { + sources := []GenerationSource{ + {Type: "prompt", Prompt: "be helpful"}, + {Type: "agent", AgentName: "support"}, + {Type: "traces", AgentName: "support"}, + } + + kept := WithoutAgentSource(sources) + + assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) + assert.Len(t, sources, 3, "the original must not be modified; it is retried from") +} + +// The retry only happens when something is left to generate from, so this is +// what stops a second billed job that would fail the same way. +func TestHasPromptSource(t *testing.T) { + assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) + assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), + "an empty prompt is nothing to generate from") + assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) + assert.False(t, HasPromptSource(nil)) +} + +// The request body is what the service validates, so the fields it keys on are +// pinned rather than left to whatever the builder happens to set. +func TestNewDataGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) + + require.NotNil(t, req) + assert.Equal(t, "support-regression", req.Inputs.Name) + assert.Equal(t, "evaluation", req.Inputs.Scenario) + assert.Equal(t, "simple_qna", req.Inputs.Options.Type) + assert.Equal(t, 15, req.Inputs.Options.MaxSamples) + assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} + +// The evaluator request sends the name twice, under two keys the service reads +// separately. Setting only one produces a job that runs and returns an +// evaluator under the wrong name. +func TestNewEvaluatorGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "grade politeness"}} + + req := NewEvaluatorGenerationJobRequest("support-quality", "gpt-4o", sources) + + require.NotNil(t, req) + assert.Equal(t, "support-quality", req.Inputs.Name) + assert.Equal(t, "support-quality", req.Inputs.EvaluatorName) + assert.Equal(t, "gpt-4o", req.Inputs.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go new file mode 100644 index 00000000000..3a71ccd4970 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/url" + "strconv" + "strings" + + "azureaieval/internal/messages" +) + +// InsightTypeEvaluationComparison compares evaluation runs. The service also +// defines EvaluationRunClusterInsight and AgentClusterInsight, which this +// extension does not use. +const InsightTypeEvaluationComparison = "EvaluationComparison" + +const pathInsights = "/insights" + +// InsightRequest is the polymorphic body the service dispatches on. `type` is +// the discriminator; without it the request is rejected because the underlying +// contract is an interface. +type InsightRequest struct { + Type string `json:"type"` + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` +} + +// CreateInsightRequest wraps the request. DisplayName is required; the service +// rejects a body without it before it looks at anything else. +type CreateInsightRequest struct { + DisplayName string `json:"displayName"` + Request *InsightRequest `json:"request"` +} + +// LenientFloat is a float64 that also decodes the quoted forms the service +// uses for values JSON cannot express. +// +// A run with a single sample has an undefined standard deviation, and the +// service sends it as the string "NaN" because JSON has no NaN literal. +// Decoding that into a plain float64 fails the entire comparison — including +// the TooFewSamples verdict that exists to explain exactly this case — so a +// one-sample gate reported a parse error instead of its result. +type LenientFloat float64 + +func (f *LenientFloat) UnmarshalJSON(data []byte) error { + s := strings.TrimSpace(string(data)) + if s == "null" { + *f = LenientFloat(math.NaN()) + return nil + } + // "NaN", "Infinity", "-Infinity" and ordinary numbers arrive quoted; + // ParseFloat accepts all of them once the quotes are gone. + if unquoted, err := strconv.Unquote(s); err == nil { + s = strings.TrimSpace(unquoted) + if s == "" { + *f = LenientFloat(math.NaN()) + return nil + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return messages.ParsingNumber(string(data), err) + } + *f = LenientFloat(v) + return nil +} + +// MarshalJSON writes non-finite values as null. encoding/json refuses to +// marshal NaN or ±Inf at all, which would turn `-o json` into an error the +// moment a comparison contained one; null is valid JSON and reads as the +// "undefined" that a one-sample standard deviation actually is. +func (f LenientFloat) MarshalJSON() ([]byte, error) { + if math.IsNaN(float64(f)) || math.IsInf(float64(f), 0) { + return []byte("null"), nil + } + return json.Marshal(float64(f)) +} + +// Defined reports whether the value is a real number that can be shown. +func (f LenientFloat) Defined() bool { + return !math.IsNaN(float64(f)) && !math.IsInf(float64(f), 0) +} + +// RunSummary is one run's aggregate for a single metric. +type RunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average LenientFloat `json:"average"` + StandardDeviation LenientFloat `json:"standardDeviation"` +} + +// CompareItem is one treatment run measured against the baseline. +type CompareItem struct { + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate LenientFloat `json:"deltaEstimate"` + PValue LenientFloat `json:"pValue"` + // TreatmentEffect classifies the result, e.g. TooFewSamples when the + // sample count cannot support a conclusion. + TreatmentEffect string `json:"treatmentEffect,omitempty"` +} + +// MetricComparison is the baseline and treatments for one testing criterion. +type MetricComparison struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + Evaluator string `json:"evaluator"` + BaselineRunSummary *RunSummary `json:"baselineRunSummary,omitempty"` + CompareItems []CompareItem `json:"compareItems,omitempty"` +} + +// InsightResult carries the comparison once the insight succeeds. +type InsightResult struct { + Comparisons []MetricComparison `json:"comparisons,omitempty"` + // Method names the statistical test, e.g. PairedTTest. + Method string `json:"method,omitempty"` + Type string `json:"type,omitempty"` + Error any `json:"error,omitempty"` +} + +// Insight is the long-running operation the comparison runs as. +type Insight struct { + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + State string `json:"state,omitempty"` + Request *InsightRequest `json:"request,omitempty"` + Result *InsightResult `json:"result,omitempty"` +} + +// Succeeded reports whether the insight finished with a result. +func (i *Insight) Succeeded() bool { + return i != nil && i.State == "Succeeded" +} + +// Terminal reports whether the insight has stopped changing. +func (i *Insight) Terminal() bool { + if i == nil { + return false + } + switch i.State { + case "", "NotStarted", "Running", "InProgress", "Queued": + return false + default: + return true + } +} + +// CreateInsight starts a comparison. +// +// The synchronous variant, POST /insights/sync, returns a 500 for this request +// shape, so the asynchronous form is the only usable one and the caller polls. +func (c *EvalClient) CreateInsight( + ctx context.Context, + request *CreateInsightRequest, + apiVersion string, +) (*Insight, error) { + return doRequestTyped[Insight]( + c, ctx, http.MethodPost, pathInsights, nil, request, apiVersion) +} + +// GetInsight reads a comparison's current state. +func (c *EvalClient) GetInsight( + ctx context.Context, + insightID string, + apiVersion string, +) (*Insight, error) { + path := fmt.Sprintf("%s/%s", pathInsights, url.PathEscape(insightID)) + return doRequestTyped[Insight](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go new file mode 100644 index 00000000000..b8db134dc79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exact body the service returned for a comparison of two single-sample +// runs. `standardDeviation` is the string "NaN" because JSON has no NaN +// literal, and decoding it into a float64 failed the whole comparison — losing +// the TooFewSamples verdict that explains the very situation that produced it. +const oneSampleComparison = `{ + "comparisons": [ + { + "testingCriteria": "task_adherence", + "metric": "task_adherence", + "evaluator": "builtin.task_adherence", + "baselineRunSummary": { + "runId": "evalrun_base", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "compareItems": [ + { + "treatmentRunSummary": { + "runId": "evalrun_treat", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "deltaEstimate": 0.0, + "pValue": 1.0, + "treatmentEffect": "TooFewSamples" + } + ] + } + ], + "method": "TTest", + "type": "EvaluationComparison" +}` + +func TestInsightResult_DecodesQuotedNaN(t *testing.T) { + var got InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &got)) + + require.Len(t, got.Comparisons, 1) + c := got.Comparisons[0] + require.NotNil(t, c.BaselineRunSummary) + + assert.Equal(t, 1.0, float64(c.BaselineRunSummary.Average)) + assert.False(t, c.BaselineRunSummary.StandardDeviation.Defined(), + "a single sample has no standard deviation") + + require.Len(t, c.CompareItems, 1) + assert.Equal(t, "TooFewSamples", c.CompareItems[0].TreatmentEffect, + "the verdict survives, which is the whole point of not failing the parse") + assert.Equal(t, 1.0, float64(c.CompareItems[0].PValue)) +} + +func TestLenientFloat_AcceptsBothShapes(t *testing.T) { + cases := map[string]func(LenientFloat) bool{ + `0.75`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"0.75"`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"NaN"`: func(f LenientFloat) bool { return !f.Defined() }, + `"Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `"-Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `null`: func(f LenientFloat) bool { return !f.Defined() }, + `""`: func(f LenientFloat) bool { return !f.Defined() }, + } + + for raw, ok := range cases { + var f LenientFloat + require.NoError(t, json.Unmarshal([]byte(raw), &f), "decoding %s", raw) + assert.True(t, ok(f), "unexpected value decoding %s", raw) + } + + var f LenientFloat + assert.Error(t, json.Unmarshal([]byte(`"not a number"`), &f), + "genuine garbage must still be reported") +} + +// encoding/json refuses to marshal NaN, so `-o json` would fail on any +// comparison holding one unless it is written as null. +func TestLenientFloat_MarshalsNonFiniteAsNull(t *testing.T) { + b, err := json.Marshal(LenientFloat(math.NaN())) + require.NoError(t, err) + assert.Equal(t, "null", string(b)) + + b, err = json.Marshal(LenientFloat(0.5)) + require.NoError(t, err) + assert.Equal(t, "0.5", string(b)) + + // The whole result has to survive a round trip, since that is what + // `results compare -o json` emits. + var res InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &res)) + out, err := json.Marshal(res) + require.NoError(t, err, "a comparison containing NaN must still emit JSON") + assert.Contains(t, string(out), `"standardDeviation":null`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go new file mode 100644 index 00000000000..9249a8e35ac --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -0,0 +1,696 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// --------------------------------------------------------------------------- +// Data Generation Jobs +// --------------------------------------------------------------------------- + +// DataGenerationJobRequest is the request body for CreateDataGenerationJob. +type DataGenerationJobRequest struct { + Inputs DataGenerationInputs `json:"inputs"` +} + +// DataGenerationInputs holds the inputs for a data generation job. +type DataGenerationInputs struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Options DataGenerationOptions `json:"options"` + Sources []GenerationSource `json:"sources"` +} + +// DataGenerationOptions holds configuration for data generation. +type DataGenerationOptions struct { + Type string `json:"type"` + MaxSamples int `json:"max_samples"` + ModelOptions ModelOptions `json:"model_options"` +} + +// ModelOptions holds the model selection for generation. +type ModelOptions struct { + Model string `json:"model"` +} + +// GenerationSource describes a source used for dataset or evaluator generation. +type GenerationSource struct { + Type string `json:"type"` + Prompt string `json:"prompt,omitempty"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` +} + +// Agent is the part of a catalog agent that describes what it does. +// +// An agent is returned with its versions inlined rather than as a list, and +// only `latest` is populated on a plain read. +type Agent struct { + Name string `json:"name"` + Versions struct { + Latest *AgentVersion `json:"latest"` + } `json:"versions"` +} + +// AgentVersion is one published revision of an agent. +type AgentVersion struct { + Version string `json:"version"` + Definition struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + } `json:"definition"` +} + +// Instructions returns the newest version's system prompt, or "" when the agent +// has no published version. +func (a *Agent) Instructions() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) +} + +// Model returns the newest version's deployment, or "" when the agent has no +// published version. It is what generation falls back to when the caller names +// no deployment of its own: the model already judged good enough to answer as +// this agent is the sensible default for writing its test cases. +func (a *Agent) Model() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Model) +} + +// GenerationJob is the response for data and evaluator generation job operations. +type GenerationJob struct { + ID string `json:"id"` + Status string `json:"status"` + Result json.RawMessage `json:"result,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// JobError captures error details from a failed generation job. +type JobError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// ResolvedNameVersion extracts the name and version from the generation job result. +// If name is empty, both return values are empty (caller should treat as no result). +// If version is empty, it defaults to "latest". +func (j *GenerationJob) ResolvedNameVersion() (string, string) { + name := j.resultStringField("name") + if name == "" { + return "", "" + } + version := j.resultStringField("version") + if version == "" { + version = "latest" + } + return name, version +} + +// resultStringField extracts a string field from the raw Result JSON. +// It first checks for a top-level key, then falls back to outputs[0].key +// to handle the nested response format. +func (j *GenerationJob) resultStringField(key string) string { + if len(j.Result) == 0 { + return "" + } + var m map[string]json.RawMessage + if err := json.Unmarshal(j.Result, &m); err != nil { + return "" + } + + // Try top-level field first. + if raw, ok := m[key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + + // Fall back to outputs[0].key for nested response format. + if rawOutputs, ok := m["outputs"]; ok { + var outputs []map[string]json.RawMessage + if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { + if raw, ok := outputs[0][key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + } + } + + return "" +} + +// --------------------------------------------------------------------------- +// Evaluator Generation Jobs +// --------------------------------------------------------------------------- + +// EvaluatorGenerationJobRequest is the request body for CreateEvaluatorGenerationJob. +type EvaluatorGenerationJobRequest struct { + Inputs EvaluatorGenerationInputs `json:"inputs"` +} + +// EvaluatorGenerationInputs holds the inputs for an evaluator generation job. +type EvaluatorGenerationInputs struct { + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + Category string `json:"category,omitempty"` + Model string `json:"model"` + Sources []GenerationSource `json:"sources"` +} + +// --------------------------------------------------------------------------- +// Evaluator Versions +// --------------------------------------------------------------------------- + +// EvaluatorVersion is the response for evaluator version operations. +type EvaluatorVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// Evaluator Definition (Rubric) +// --------------------------------------------------------------------------- + +// EvaluatorResult is the top-level response from evaluator generation, +// containing the evaluator's definition. +type EvaluatorResult struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Definition EvaluatorDefinition `json:"definition"` +} + +// EvaluatorDefinition describes an evaluator's scoring rubric. +type EvaluatorDefinition struct { + Type string `json:"type"` + Dimensions []EvaluatorDimension `json:"dimensions"` +} + +// EvaluatorDimension is a single scoring dimension within a rubric evaluator. +type EvaluatorDimension struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Weight int `json:"weight"` + AlwaysApplicable bool `json:"always_applicable,omitempty"` +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// CreateDatasetRequest is the request body for CreateDataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// OpenAI Evals +// --------------------------------------------------------------------------- + +// DataSourceConfig describes the data source for an OpenAI eval. +type DataSourceConfig struct { + Type string `json:"type"` + ItemSchema map[string]any `json:"item_schema"` + IncludeSampleSchema bool `json:"include_sample_schema"` +} + +// DataSourceSchema defines the item and sample schemas for an eval data source. +type DataSourceSchema struct { + Item map[string]any `json:"item,omitempty"` + Sample map[string]any `json:"sample,omitempty"` +} + +// TestingCriterion describes a single evaluator in testing_criteria. +type TestingCriterion struct { + Type string `json:"type"` + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + EvaluatorVersion string `json:"evaluator_version,omitempty"` + InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` +} + +// CreateOpenAIEvalRequest is the request body for CreateOpenAIEval. +type CreateOpenAIEvalRequest struct { + Name string `json:"name"` + Metadata map[string]string `json:"metadata,omitempty"` + DataSourceConfig *DataSourceConfig `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` +} + +// UpdateOpenAIEvalRequest is UpdateEvalParametersBody: the only fields an eval +// accepts after creation. Testing criteria and the data source are fixed at +// create time, and the service drops anything else here silently rather than +// rejecting it. +type UpdateOpenAIEvalRequest struct { + Name string `json:"name,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// OpenAIEval is the response for an OpenAI eval definition. +type OpenAIEval struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + // The service returns these on a read, and they are what makes `show` a + // definition rather than an id: what the eval grades, and where its rows + // come from. Both are fixed at creation. + DataSourceConfig map[string]any `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` +} + +// OpenAIEvalList is the response for listing OpenAI eval definitions. +type OpenAIEvalList struct { + Data []OpenAIEval `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor, read the same + // way OutputItemList reads them: only when present, so a service that sends + // neither still yields one page. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` +} + +// --------------------------------------------------------------------------- +// OpenAI Eval Runs +// --------------------------------------------------------------------------- + +// CreateOpenAIEvalRunRequest is the request body for CreateOpenAIEvalRun. +type CreateOpenAIEvalRunRequest struct { + Name string `json:"name"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// EvalRunDataSourceType defines the type for an eval run data source. +type EvalRunDataSourceType string + +const ( + // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. + EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" + + // EvalRunDataSourceTypeTraces evaluates an agent's recorded traces instead of + // a dataset. The service reads them from Application Insights, so the agent + // must be emitting gen_ai.input.messages / gen_ai.output.messages. + // + // Deprecated in favour of EvalRunDataSourceTypeTracePreview, which is the + // only shape that carries an agent version. This one silently discards + // agent_version and start_time and re-imposes its own lookback. + EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" + + // EvalRunDataSourceTypeTracePreview is the shape that honours what the + // caller asks for: a pinned agent version, and a window with both bounds. + EvalRunDataSourceTypeTracePreview EvalRunDataSourceType = "azure_ai_trace_data_source_preview" + + // EvalRunDataSourceTypeResponses evaluates responses the project already + // stored, addressed by id. + EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" + + // EvalRunDataSourceTypeJSONL scores the rows as they are, invoking nothing. + EvalRunDataSourceTypeJSONL EvalRunDataSourceType = "jsonl" +) + +// EvalRunDataContentType defines the source type for eval run data content. +type EvalRunDataContentType string + +const ( + EvalRunDataContentTypeFileContent EvalRunDataContentType = "file_content" + EvalRunDataContentTypeFileID EvalRunDataContentType = "file_id" +) + +// EvalRunDataSource describes the data source for an eval run with agent target completions. +type EvalRunDataSource struct { + Type EvalRunDataSourceType `json:"type"` + InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` + Target *EvalRunTarget `json:"target,omitempty"` + + // Traces only. The window is expressed as a lookback in hours, not as a + // start bound: the service has no start_time on this data source and + // silently falls back to its default when one is sent. + AgentName string `json:"agent_name,omitempty"` + LookbackHours int `json:"lookback_hours,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` + + // Trace preview only. Everything it carries is nested in the filter, which + // is where the service reads it. + TraceSource *TraceSourceFilter `json:"trace_source,omitempty"` + + // Responses only. + ItemGenerationParams *ItemGenerationParams `json:"item_generation_params,omitempty"` +} + +// ItemGenerationParams says how the service should turn a source into the items +// it evaluates. +type ItemGenerationParams struct { + Type string `json:"type"` + MaxNumTurns int `json:"max_num_turns,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` +} + +// EvalRunInputMessages describes how input messages are constructed from dataset items. +type EvalRunInputMessages struct { + Type string `json:"type"` + Template []EvalRunMessageTemplate `json:"template"` +} + +// EvalRunMessageTemplate describes a single message in the input template. +type EvalRunMessageTemplate struct { + Role string `json:"role"` + Content string `json:"content"` + Type string `json:"type"` +} + +// EvalRunTarget describes what the run invokes: an agent by name, or a model +// deployment directly. Only the fields belonging to Type are sent. +type EvalRunTarget struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Version *string `json:"version,omitempty"` + ToolDescriptions []string `json:"tool_descriptions,omitempty"` + Model string `json:"model,omitempty"` +} + +// EvalRunDataContent holds the source reference within an EvalRunDataSource. +type EvalRunDataContent struct { + Type EvalRunDataContentType `json:"type"` + ID string `json:"id,omitempty"` + Content []map[string]any `json:"content,omitempty"` +} + +// NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. +// The rows must be supplied separately via SetFileContent. +func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_agent", + Name: agentName, + Version: agentVersion, + ToolDescriptions: []string{}, + }, + } +} + +// TraceSourceFilter selects which spans a trace run reads. +// +// The window and the cap live here, not beside the data source type: verified +// against the service, which echoes them back only from inside this object and +// silently ignores them at the top level. +type TraceSourceFilter struct { + Type string `json:"type"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` +} + +// NewTracePreviewDataSource builds the shape that keeps what the caller sent. +// +// The legacy azure_ai_traces source drops agent_version and start_time without +// saying so and re-applies its own lookback, so a redeployed agent was +// evaluated against whichever version the service picked. +func NewTracePreviewDataSource( + agentName, agentVersion string, + start, end time.Time, + maxTraces int, +) *EvalRunDataSource { + filter := &TraceSourceFilter{ + Type: "agent_filter", + AgentName: agentName, + AgentVersion: agentVersion, + MaxTraces: maxTraces, + } + if !start.IsZero() { + filter.StartTime = start.Unix() + } + if !end.IsZero() { + filter.EndTime = end.Unix() + } + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeTracePreview, + TraceSource: filter, + } +} + +// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. +// +// Used when an eval declares no target: the rows already hold both sides of the +// exchange, which is how a recorded conversation is evaluated. +func NewDatasetOnlyDataSource() *EvalRunDataSource { + return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} +} + +// NewModelTargetDataSource sends the dataset's questions straight to a model +// deployment, with no agent in front of it. +// +// The model answers as plain text, so an eval evaluating one has to bind its +// response to {{sample.output_text}} rather than the richer output an agent +// produces. +func NewModelTargetDataSource(model string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_model", + Model: model, + }, + } +} + +// NewResponsesDataSource evaluates responses the project already stored. +// +// The ids travel as ordinary JSONL rows and a data_mapping points the service +// at the field holding each one, which is how it retrieves the chat history +// behind the response. +func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { + rows := make([]map[string]any, 0, len(responseIDs)) + for _, id := range responseIDs { + rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) + } + + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeResponses, + ItemGenerationParams: &ItemGenerationParams{ + Type: "response_retrieval", + MaxNumTurns: maxTurns, + DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, + Source: &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: rows, + }, + }, + } +} + +// SetFileContent sets the data source to use inline file content. +// +// There is no by-reference counterpart. A run's `file_id` means an uploaded +// file, and a dataset name is not one — sending it is rejected with "invalid +// data source file ids" — so registered datasets are fetched and sent inline +// too. See readRegisteredDataset. +func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: items, + } +} + +// OpenAIEvalRun is the response for an OpenAI eval run. +type OpenAIEvalRun struct { + ID string `json:"id"` + EvalID string `json:"eval_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + ReportURL string `json:"report_url,omitempty"` + // PortalURL is built by the extension, not returned by the service, so that + // `-o json` carries the same link the terminal prints. + PortalURL string `json:"portal_url,omitempty"` + + // Result summary + ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` + PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Failure returns why the run failed, or "" when it did not. +// +// The field is always present and its members are null on success, so its +// presence says nothing on its own. +func (r *OpenAIEvalRun) Failure() string { + if r == nil || r.Error == nil { + return "" + } + return strings.TrimSpace(r.Error.Message) +} + +// EvalRunResultCounts holds pass/fail/error/skip counts for a run. +type EvalRunResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// EvalRunCriteriaResult holds per-testing-criteria pass/fail counts. +type EvalRunCriteriaResult struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// OpenAIEvalRunList is the response for listing OpenAI eval runs. +type OpenAIEvalRunList struct { + Data []OpenAIEvalRun `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` +} + +// OutputItemList is a page of a run's per-sample results. +type OutputItemList struct { + Data []OutputItem `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor. They are only + // read, never required: a service that returns neither yields one page, + // which is what this client did before it could see them at all. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` +} + +// OutputItem is one evaluated row: the dataset item, and every evaluator's +// verdict on it. +type OutputItem struct { + ID string `json:"id"` + RunID string `json:"run_id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item,omitempty"` + Results []OutputResult `json:"results,omitempty"` +} + +// OutputResult is one evaluator's verdict on one row. +type OutputResult struct { + Name string `json:"name"` + Metric string `json:"metric,omitempty"` + Score LenientFloat `json:"score"` + Label string `json:"label,omitempty"` + // Passed is a pointer because an absent verdict and a failing one are + // different claims. As a plain bool a result the service sent without one -- + // an evaluator that errored on this row -- read as a definite "fail", which + // names the evaluator as the thing that judged badly rather than the thing + // that did not run. + Passed *bool `json:"passed"` + // Reason is the judge's explanation, which is the part a failing row is + // actually looked at for. + Reason string `json:"reason,omitempty"` +} + +// Failed reports whether this row is one to look at: any evaluator failed it, +// did not judge it, or it produced no verdict at all. +// +// A row that errored badly enough to carry no results used to answer false, so +// --failed-only hid it -- and that filter is exactly where someone looks to +// find out what went wrong. A result carrying no verdict is the same absence +// one level down. +func (o OutputItem) Failed() bool { + if len(o.Results) == 0 { + return true + } + for _, r := range o.Results { + if !r.DidPass() { + return true + } + } + return false +} + +// DidPass reports whether this result is a recorded pass. An absent verdict is +// not one. +func (r OutputResult) DidPass() bool { + return r.Passed != nil && *r.Passed +} + +// Judged reports whether the evaluator returned a verdict at all. +func (r OutputResult) Judged() bool { + return r.Passed != nil +} + +// Input renders the row's own columns for display, leaving out the +// service-injected `sample.*` bindings and the plumbing ids, which are not what +// the dataset author wrote. +func (o OutputItem) Input() string { + if len(o.DataSourceItem) == 0 { + return "" + } + skip := map[string]bool{ + "response_id": true, "agent_id": true, "agent_name": true, + "agent_version": true, "conversation_id": true, + "previous_response_id": true, "trace_id": true, "span_id": true, + } + keys := make([]string, 0, len(o.DataSourceItem)) + for k := range o.DataSourceItem { + if skip[k] || strings.HasPrefix(k, "sample.") { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%v", k, o.DataSourceItem[k])) + } + return strings.Join(parts, " ") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go new file mode 100644 index 00000000000..94da5781dc4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -0,0 +1,702 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefixes for eval service endpoints. +const ( + pathDataGenerationJobs = "/data_generation_jobs" + pathEvaluatorGenerationJobs = "/evaluator_generation_jobs" + pathEvaluators = "/evaluators" + pathDatasets = "/datasets" + pathOpenAIEvals = "/openai/v1/evals" + pathAgents = "/agents" +) + +// EvalClient provides methods for interacting with the Azure AI eval APIs. +type EvalClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewEvalClient creates a new EvalClient. +func NewEvalClient(endpoint string, cred azcore.TokenCredential) *EvalClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-evals", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewEvalClientFromPipeline creates an EvalClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewEvalClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *EvalClient { + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataGenerationJob starts a dataset generation job for eval onboarding. +func (c *EvalClient) CreateDataGenerationJob( + ctx context.Context, + request *DataGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathDataGenerationJobs, nil, request, apiVersion) +} + +// GetDataGenerationJob gets the current state of a dataset generation job. +func (c *EvalClient) GetDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorGenerationJob starts an evaluator generation job for eval onboarding. +func (c *EvalClient) CreateEvaluatorGenerationJob( + ctx context.Context, + request *EvaluatorGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathEvaluatorGenerationJobs, nil, request, apiVersion) +} + +// GetEvaluatorGenerationJob gets the current state of an evaluator generation job. +func (c *EvalClient) GetEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathEvaluatorGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GenerationJobList is the listing envelope both job types answer with. It is +// `data`, not the `value` the dataset and evaluator routes use, and it carries +// the same has_more/last_id cursor as the OpenAI listings. +// +// The cursor fields were missing, so both listings read one page and stopped: +// a project with more than a page of jobs answered `job list` with the first +// twenty and no sign there were more. +type GenerationJobList struct { + Data []GenerationJob `json:"data"` + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` +} + +// ListDataGenerationJobs returns the project's dataset generation jobs. +func (c *EvalClient) ListDataGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return c.listGenerationJobs(ctx, pathDataGenerationJobs, apiVersion) +} + +// ListEvaluatorGenerationJobs returns the project's evaluator generation jobs. +func (c *EvalClient) ListEvaluatorGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return c.listGenerationJobs(ctx, pathEvaluatorGenerationJobs, apiVersion) +} + +// listGenerationJobs is the walk both job listings share. +func (c *EvalClient) listGenerationJobs( + ctx context.Context, + path string, + apiVersion string, +) (*GenerationJobList, error) { + all := &GenerationJobList{} + err := collectPages(0, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, path, query, nil, apiVersion) + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err + } + return all, nil +} + +// CancelDataGenerationJob stops a dataset generation job. +func (c *EvalClient) CancelDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// CancelEvaluatorGenerationJob stops an evaluator generation job. +func (c *EvalClient) CancelEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// cancelGenerationJob posts to the colon form of the route. +// +// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while +// `{id}:cancel` reaches the action. The empty object is what carries a content +// type, without which the route answers 415. +func (c *EvalClient) cancelGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) (*GenerationJob, error) { + path := basePath + "/" + url.PathEscape(operationID) + ":cancel" + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// DeleteDataGenerationJob removes a dataset generation job record. +func (c *EvalClient) DeleteDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// DeleteEvaluatorGenerationJob removes an evaluator generation job record. +func (c *EvalClient) DeleteEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// deleteGenerationJob discards the job record. The artifact the job produced is +// already registered and is not affected. +func (c *EvalClient) deleteGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) error { + path := basePath + "/" + url.PathEscape(operationID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// GetAgent reads an agent from the project's catalog. +// +// Only the newest version is returned, which is the one generation is seeded +// from: the point is to describe what the agent does now. +func (c *EvalClient) GetAgent( + ctx context.Context, + name string, + apiVersion string, +) (*Agent, error) { + path := pathAgents + "/" + url.PathEscape(name) + return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorVersion creates a new version of a named evaluator. +// The body should be the full evaluator JSON with the definition field updated. +// +// previous is the evaluator document the caller has already read, or nil when +// it read none. It is what keeps the publish from being answered with the +// version that document holds. +func (c *EvalClient) CreateEvaluatorVersion( + ctx context.Context, + name string, + body json.RawMessage, + previous json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + return c.publishEvaluatorVersion(ctx, name, previous, apiVersion, func() (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + }) +} + +// versionSettle bounds the wait for the service to start assigning the next +// version number. +const ( + versionSettleTimeout = 45 * time.Second + versionSettleInterval = 3 * time.Second + versionSettleAge = 8 * time.Second +) + +// publishedVersion is the little of an evaluator document this needs: which +// version it is, and when it was written. +type publishedVersion struct { + Version string `json:"version"` + ModifiedAt time.Time `json:"modified_at"` + CreatedAt time.Time `json:"created_at"` +} + +// writtenAt reports when the version was last written, preferring the +// modification time and falling back to creation. +func (p publishedVersion) writtenAt() time.Time { + if !p.ModifiedAt.IsZero() { + return p.ModifiedAt + } + return p.CreatedAt +} + +// publishEvaluatorVersion publishes and then makes sure a new version is what +// came back. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over that version's contents instead +// of adding one. It is a race rather than a fixed window -- a second publish +// has been seen both colliding a quarter of a second later and succeeding +// immediately -- and nothing observable marks its end. +// +// That matters because versions are the unit an eval binds to. `evaluator +// create` followed by `evaluator update`, which is what a first authoring +// session looks like, would otherwise leave one version holding the second +// definition and every eval bound to the first silently scoring against a +// rubric nobody chose. +// +// So there are two defenses. The publish is held back until the version the +// caller read has had time to settle, which is what keeps the collision from +// happening at all; and the version that comes back is checked, which is what +// keeps a collision that happens anyway from being reported as success. The +// recheck republishes the same body, so it cannot make a collision worse than +// the first attempt already did. +// +// What the caller reads is used rather than the version listing because the +// listing lags a publish too: asked immediately after a create it answers 404, +// so a guard that trusted it would stand down in exactly the case it exists +// for. Callers that publish an evaluator have already read it to decide +// between creating and updating. +func (c *EvalClient) publishEvaluatorVersion( + ctx context.Context, + name string, + previous json.RawMessage, + apiVersion string, + publish func() (*EvaluatorVersion, error), +) (*EvaluatorVersion, error) { + var known publishedVersion + if len(previous) > 0 { + _ = json.Unmarshal(previous, &known) + } + + latest := parseVersionNumber(known.Version) + if listed := c.LatestEvaluatorVersionNumber(ctx, name, apiVersion); listed > latest { + latest = listed + } + + if written := known.writtenAt(); !written.IsZero() { + if wait := versionSettleAge - time.Since(written); wait > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + } + } + + deadline := time.Now().Add(versionSettleTimeout) + for { + created, err := publish() + if err != nil { + return nil, err + } + if latest == 0 || parseVersionNumber(created.Version) > latest { + return created, nil + } + if time.Now().After(deadline) { + return nil, messages.EvaluatorVersionNotAdvancing( + name, created.Version, versionSettleTimeout) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(versionSettleInterval): + } + } +} + +// GetEvaluatorRaw gets an evaluator by name and version as raw JSON. +// If version is empty, the latest version is resolved first. +// +// The service has no route for an unversioned evaluator: GET +// /evaluators/{name} returns 404 with no body, so the version cannot simply be +// left off the path. +func (c *EvalClient) GetEvaluatorRaw( + ctx context.Context, + name string, + version string, + apiVersion string, +) (json.RawMessage, error) { + if version == "" { + latest, err := c.LatestEvaluatorVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + version = latest + } + path := pathEvaluators + "/" + url.PathEscape(name) + + "/versions/" + url.PathEscape(version) + return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// LatestEvaluatorVersion returns the newest registered version of an evaluator. +func (c *EvalClient) LatestEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) (string, error) { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil { + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", messages.EvaluatorHasNoVersions(name) + } + latest := pickLatestVersion(list.Value) + if latest == "" { + return "", messages.EvaluatorHasNoUsableVersion(name) + } + return latest, nil +} + +// pickLatestVersion selects the highest evaluator version. +// +// Versions are integers rendered as strings, so they are compared numerically: +// a lexical compare would rank "9" above "15", and the service already +// publishes evaluators at version 15 and 17. A non-numeric version is used +// only when nothing numeric is present. +func pickLatestVersion(entries []EvaluatorSummary) string { + best := "" + bestNum := -1 + for _, entry := range entries { + if entry.Version == "" { + continue + } + num, err := strconv.Atoi(entry.Version) + if err != nil { + if best == "" { + best = entry.Version + } + continue + } + if num > bestNum { + bestNum, best = num, entry.Version + } + } + return best +} + +// CreateOpenAIEval creates an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEval( + ctx context.Context, + request *CreateOpenAIEvalRequest, +) (*OpenAIEval, error) { + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, pathOpenAIEvals, nil, request, "") +} + +// ListOpenAIEvals lists OpenAI eval definitions. +func (c *EvalClient) ListOpenAIEvals(ctx context.Context, limit int) (*OpenAIEvalList, error) { + all := &OpenAIEvalList{} + err := collectPages(limit, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[OpenAIEvalList]( + c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err + } + return all, nil +} + +// collectPages walks an OpenAI-shaped listing until the service stops offering +// a cursor, or until limit rows have been gathered. +// +// A listing that stops at the first page is a silent wrong answer rather than a +// short one: "is this name ambiguous?" and "which run is newest?" are both +// decided from these rows, so a second page nobody asked for turns a refusal +// into a wrong choice. fetch reports how many rows it added and the cursor it +// was given, so the two listings share this loop instead of a third copy. +func collectPages( + limit int, + fetch func(query map[string]string) (added int, hasMore bool, lastID string, err error), +) error { + gathered := 0 + after := "" + // A cursor that keeps returning rows while pointing back at itself would + // spin forever, holding the command open and growing the slice until the + // process dies. The next-link walker in pages.go bounds itself the same + // way; the cursor listings simply never did. + seen := map[string]bool{} + for range maxPages { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit - gathered) + } + if after != "" { + query["after"] = after + } + + added, hasMore, lastID, err := fetch(query) + if err != nil { + return err + } + gathered += added + + if !hasMore || lastID == "" || added == 0 { + return nil + } + if limit > 0 && gathered >= limit { + return nil + } + if seen[lastID] { + log.Printf("[eval_api] cursor %q repeated; the listing may be incomplete", lastID) + return nil + } + seen[lastID] = true + after = lastID + } + log.Printf("[eval_api] stopped after %d pages; the listing may be incomplete", maxPages) + return nil +} + +// GetOpenAIEval gets an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") +} + +// DeleteOpenAIEval removes an eval definition and its runs. +func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// UpdateOpenAIEval edits an eval in place. The route is a POST on the eval +// itself, matching how this surface spells run cancel -- there is no PATCH verb +// here. +// +// Only what UpdateEvalParametersBody reaches is editable: name, metadata and +// properties. Anything else the service drops silently, so substance never +// travels through this call and an edit that touches it is a new eval. +func (c *EvalClient) UpdateOpenAIEval( + ctx context.Context, + evalID string, + request *UpdateOpenAIEvalRequest, +) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, path, nil, request, "") +} + +// CreateOpenAIEvalRun starts a run for an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEvalRun( + ctx context.Context, + evalID string, + request *CreateOpenAIEvalRunRequest, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, request, "") +} + +// ListOpenAIEvalRuns lists runs for an OpenAI eval definition. +func (c *EvalClient) ListOpenAIEvalRuns( + ctx context.Context, + evalID string, + limit int, +) (*OpenAIEvalRunList, error) { + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + all := &OpenAIEvalRunList{} + err := collectPages(limit, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[OpenAIEvalRunList]( + c, ctx, http.MethodGet, path, query, nil, "") + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err + } + return all, nil +} + +// GetOpenAIEvalRun gets a run for an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodGet, path, nil, nil, "") +} + +func (c *EvalClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + return c.doRequestWithHeaders(ctx, method, path, query, body, apiVersion, nil) +} + +// doRequestWithHeaders is doRequest with extra request headers, which the +// preview evaluator operations need to opt in to the properties they set. +func (c *EvalClient) doRequestWithHeaders( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, + headers map[string]string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + + // Callers escape the ids they interpolate, so the path is set as the raw + // one. Assigning it to u.Path re-escapes the percent signs, and an + // evaluator named "my evaluator" then addresses one named "my%20evaluator". + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, messages.InvalidRequestPath(escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + for k, v := range headers { + req.Raw().Header.Set(k, v) + } + + log.Printf("[eval_api] %s %s", method, urlsafe.URL(u)) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, messages.MarshalingRequest(err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, messages.SettingRequestBody(err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + + log.Printf("[eval_api] response status: %d", resp.StatusCode) + + // 204 belongs here: a delete that removed the resource answers No Content, + // and treating that as a failure reports every successful delete as an + // error. doRequestTyped already tolerates the empty body. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + // Restore the body so runtime.NewResponseError can read it. + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *EvalClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, messages.ParsingResponse(err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go new file mode 100644 index 00000000000..da7ae585717 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// call is what the client actually put on the wire, which is the part of these +// operations that can be wrong without anything failing to compile. +type call struct { + method string + path string + // rawPath is the path as it went over the wire, where escaping is still + // visible. path has been decoded and cannot tell %2F from a separator. + rawPath string + query url.Values + body string +} + +// recorder answers every request with status and body, remembering the last one. +func recorder(t *testing.T, status int, body string) (*EvalClient, *call) { + t.Helper() + var last call + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + last = call{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + query: r.URL.Query(), + body: string(raw), + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + }) + return client, &last +} + +// The cancel route takes a colon, not a path segment. `{id}/cancel` is a 404 +// while `{id}:cancel` reaches the action, and nothing but the URL says so. +func TestCancelGenerationJob_UsesTheColonForm(t *testing.T) { + tests := []struct { + name string + cancel func(*EvalClient) error + want string + }{ + { + name: "dataset", + cancel: func(c *EvalClient) error { + _, err := c.CancelDataGenerationJob(context.Background(), "dgj_1", "v1") + return err + }, + want: "/data_generation_jobs/dgj_1:cancel", + }, + { + name: "evaluator", + cancel: func(c *EvalClient) error { + _, err := c.CancelEvaluatorGenerationJob(context.Background(), "egj_1", "v1") + return err + }, + want: "/evaluator_generation_jobs/egj_1:cancel", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"j_1","status":"cancelled"}`) + + require.NoError(t, tt.cancel(client)) + + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, tt.want, last.path) + assert.Equal(t, "{}", last.body, + "the empty object is what carries a content type; without it the route answers 415") + }) + } +} + +// A delete that removed the record answers 204 with no body. Treating that as a +// failure would report every successful delete as an error. +func TestDeleteGenerationJob_AcceptsNoContent(t *testing.T) { + client, last := recorder(t, http.StatusNoContent, "") + + require.NoError(t, client.DeleteDataGenerationJob(context.Background(), "dgj_1", "v1")) + + assert.Equal(t, http.MethodDelete, last.method) + assert.Equal(t, "/data_generation_jobs/dgj_1", last.path) +} + +// The job routes answer with `data`, not the `value` the dataset and evaluator +// routes use. Reading the wrong key returns an empty list from a full response. +func TestListGenerationJobs_ReadsTheDataEnvelope(t *testing.T) { + client, _ := recorder(t, http.StatusOK, + `{"data":[{"id":"dgj_1","status":"completed"},{"id":"dgj_2","status":"running"}]}`) + + list, err := client.ListDataGenerationJobs(context.Background(), "v1") + + require.NoError(t, err) + require.Len(t, list.Data, 2) + assert.Equal(t, "dgj_1", list.Data[0].ID) +} + +// An id goes into the path, so one containing a separator has to be escaped or +// it silently addresses a different route. +// +// The assertion is on the wire form: the decoded path shows the separators +// again, so it cannot tell a correctly escaped id from an unescaped one. +func TestOperations_EscapeIdsInThePath(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") + + require.NoError(t, err) + assert.Equal(t, "/data_generation_jobs/dgj%2F..%2Fevil", last.rawPath, + "the id stays one segment; escaping it twice would send %252F and address a differently named job") +} + +// The api-version is what selects the contract; sending the wrong one, or none, +// is answered by a different shape than the client parses. +func TestOperations_SendTheApiVersion(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "2025-11-15-preview") + + require.NoError(t, err) + assert.Equal(t, "2025-11-15-preview", last.query.Get("api-version")) +} + +// The OpenAI-compatible eval routes send no api-version at all, so adding one +// would be as wrong as omitting it elsewhere. +func TestOpenAIEvalRoutes_SendNoApiVersion(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"eval_1"}`) + + _, err := client.GetOpenAIEval(context.Background(), "eval_1") + + require.NoError(t, err) + assert.Equal(t, "/openai/v1/evals/eval_1", last.path) + assert.Empty(t, last.query.Get("api-version")) +} + +// A rename is pushed in place so the eval keeps its id and its run history. +// The route is a POST to the eval itself, not a PATCH and not a new eval. +func TestUpdateOpenAIEval_PostsToTheEval(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"eval_1","name":"renamed"}`) + + _, err := client.UpdateOpenAIEval(context.Background(), "eval_1", + &UpdateOpenAIEvalRequest{Name: "renamed"}) + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/openai/v1/evals/eval_1", last.path) + + var sent map[string]any + require.NoError(t, json.Unmarshal([]byte(last.body), &sent)) + assert.Equal(t, "renamed", sent["name"]) +} + +// Only the newest agent version seeds generation: the point is to describe what +// the agent does now. +func TestGetAgent_ReadsTheCatalogEntry(t *testing.T) { + client, last := recorder(t, http.StatusOK, + `{"name":"support","versions":{"latest":{"version":"3",`+ + `"definition":{"instructions":"Be helpful."}}}}`) + + agent, err := client.GetAgent(context.Background(), "support", "v1") + + require.NoError(t, err) + assert.Equal(t, "/agents/support", last.path) + assert.Equal(t, "Be helpful.", agent.Instructions()) +} + +// A 404 has to arrive as one, because the commands branch on it to tell "no +// such thing" apart from "the call failed". +func TestOperations_NotFoundIsRecognizable(t *testing.T) { + client, _ := recorder(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetOpenAIEval(context.Background(), "eval_missing") + + require.Error(t, err) + assert.True(t, IsNotFound(err), "the commands branch on this to name the thing that is missing") +} + +// An empty body on a success is not a parse failure: a 204 carries none, and +// the typed helper has to hand back a zero value rather than an error. +func TestOperations_EmptyBodyIsNotAnError(t *testing.T) { + client, _ := recorder(t, http.StatusOK, "") + + job, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") + + require.NoError(t, err) + require.NotNil(t, job) + assert.Empty(t, job.ID) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go new file mode 100644 index 00000000000..7a1a7be6b41 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// --failed-only is where someone looks to find out what went wrong, so a row +// that errored badly enough to carry no verdict at all has to appear there. It +// used to answer false and be hidden. +func TestARowWithNoVerdictCountsAsFailed(t *testing.T) { + assert.True(t, OutputItem{ID: "item_1", Status: "errored"}.Failed(), + "nothing graded this row, so nothing passed it") + assert.True(t, OutputItem{ID: "item_2", Results: []OutputResult{}}.Failed(), + "an empty result set is the same absence") + assert.True(t, + OutputItem{ID: "item_3", Results: []OutputResult{{Name: "relevance"}}}.Failed(), + "a result the evaluator never judged is that absence one level down") +} + +// The ordinary cases have to keep answering as they did. +func TestFailedReadsEveryVerdict(t *testing.T) { + passing := OutputItem{Results: []OutputResult{ + {Name: "relevance", Passed: new(true)}, + {Name: "coherence", Passed: new(true)}, + }} + assert.False(t, passing.Failed(), "every evaluator passed it") + + mixed := OutputItem{Results: []OutputResult{ + {Name: "relevance", Passed: new(true)}, + {Name: "coherence", Passed: new(false)}, + }} + assert.True(t, mixed.Failed(), "one failing evaluator is enough") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go new file mode 100644 index 00000000000..bb6a6516254 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run's rows are what `run output list`, `--failed-only` and `export` all +// read, and what the mean score per evaluator is averaged over. Stopping at +// the first page would report a sample of the run as the run, with nothing on +// screen to say rows were missing. +func TestListOutputItemsFollowsTheCursor(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + n := requests.Add(1) + after := r.URL.Query().Get("after") + + switch n { + case 1: + assert.Empty(t, after, "the first page is not asked for by cursor") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{ + {"id": "a", "status": "completed"}, + {"id": "b", "status": "completed"}, + }, + "has_more": true, + "last_id": "b", + }) + case 2: + assert.Equal(t, "b", after, "the next page is asked for from the last id") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{{"id": "c", "status": "completed"}}, + "has_more": false, + }) + default: + t.Errorf("asked for a page after the service said there were none") + http.Error(w, "unexpected page request", http.StatusInternalServerError) + } + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + require.Len(t, list.Data, 3, "every page's rows belong to the run") + assert.Equal(t, []string{"a", "b", "c"}, ids(list.Data)) + assert.EqualValues(t, 2, requests.Load()) +} + +// A service that answers one page and says nothing about more is the shape +// this client was written against, and must still work exactly as before. +func TestListOutputItemsStopsWithoutACursor(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + writeJSON(t, w, map[string]any{ + "data": []map[string]any{{"id": "only", "status": "completed"}}, + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + require.Len(t, list.Data, 1) + assert.EqualValues(t, 1, requests.Load(), "one page, one request") +} + +// --limit is a cap on rows, not on requests: fetching past it would spend the +// caller's time on rows they said they did not want. +func TestListOutputItemsHonoursTheLimitAcrossPages(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + writeJSON(t, w, map[string]any{ + "data": []map[string]any{ + {"id": fmt.Sprintf("row-%d", requests.Load()), "status": "completed"}, + }, + "has_more": true, + "last_id": fmt.Sprintf("row-%d", requests.Load()), + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 2) + + require.NoError(t, err) + require.Len(t, list.Data, 2) + assert.EqualValues(t, 2, requests.Load(), "the cap stops the paging") +} + +// A page that claims more but carries nothing would otherwise loop forever. +func TestListOutputItemsStopsOnAnEmptyPage(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + assert.Less(t, requests.Load(), int32(5), "the client is looping") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{}, + "has_more": true, + "last_id": "x", + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + assert.Empty(t, list.Data) + assert.EqualValues(t, 1, requests.Load()) +} + +// writeJSON is called from the server's goroutine, so it asserts rather than +// requires: FailNow off the test's own goroutine aborts mid-response and the +// failure lands on whichever test happens to be running. +func writeJSON(t *testing.T, w http.ResponseWriter, body any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + assert.NoError(t, json.NewEncoder(w).Encode(body)) +} + +func ids(items []OutputItem) []string { + out := make([]string, 0, len(items)) + for _, i := range items { + out = append(out, i.ID) + } + return out +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go new file mode 100644 index 00000000000..70cb023e17f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +// maxPages bounds a walk the service controls. +// +// A nextLink that points at itself, or a service that keeps offering one, +// would otherwise spin forever holding the command open. The listings this +// walks are catalogues, not run output, so the bound is generous enough that +// reaching it means something is wrong rather than that a project is large. +const maxPages = 100 + +// followNextLink fetches one service-supplied page URL. +// +// The URL arrives in a response body, so it is checked against the endpoint +// before it is used: this client sends an Authorization header, and following +// a body-supplied link to another host would send the token there. +func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { + parsed, err := url.Parse(nextLink) + if err != nil { + return nil, messages.InvalidNextLink(nextLink, err) + } + base, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + + // A nextLink is allowed to be relative, and a relative one carries no host + // or scheme of its own. Resolving it against the endpoint first keeps the + // origin check meaningful instead of refusing a legitimate link. + next := base.ResolveReference(parsed) + if !sameService(base, next) { + return nil, messages.PageLinkLeftTheService(base.Host, next.Host) + } + + req, err := runtime.NewRequest(ctx, http.MethodGet, next.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + log.Printf("[eval_api] GET %s", urlsafe.URL(next)) + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + if !runtime.HasStatusCode(resp, http.StatusOK) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + return respBody, nil +} + +// sameService reports whether a page link stays on the host the client was +// pointed at. Scheme is compared too, so a link cannot downgrade to http. +func sameService(base, next *url.URL) bool { + return strings.EqualFold(base.Host, next.Host) && strings.EqualFold(base.Scheme, next.Scheme) +} + +// walkNextLinks gathers every page of an ARM-shaped listing. +// +// These listings answer with one page and a nextLink, and the link was decoded +// and dropped. That is a silent wrong answer rather than a short one: the +// evaluator listings settle which version is latest, and a version sitting on +// page two makes the answer an older one. +func walkNextLinks[T any]( + ctx context.Context, + c *EvalClient, + first *T, + nextLinkOf func(*T) string, + merge func(into, page *T), +) (*T, error) { + seen := map[string]bool{} + link := nextLinkOf(first) + for link != "" { + // A repeated link, not just a self-referencing one, ends the walk: a + // two-page cycle would otherwise run to maxPages for no benefit. + if seen[link] || len(seen) >= maxPages { + fmt.Fprint(os.Stderr, messages.Warning(messages.ListingTruncated(len(seen)))) + break + } + seen[link] = true + + body, err := c.followNextLink(ctx, link) + if err != nil { + return nil, err + } + var page T + if len(body) > 0 { + if err := json.Unmarshal(body, &page); err != nil { + return nil, messages.ParsingResponse(err) + } + } + merge(first, &page) + link = nextLinkOf(&page) + } + return first, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go new file mode 100644 index 00000000000..90dee0fd46f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func clientServing(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewEvalClientFromPipeline(srv.URL, pipeline) +} + +// A listing that stops at the first page is a silent wrong answer rather than a +// short one. resolveEvalRef decides "is this name ambiguous?" from these rows, +// so a duplicate sitting on page two turns a refusal into a wrong choice, and +// `run start` then grades against a definition the caller did not mean. +func TestListOpenAIEvalsFollowsTheCursor(t *testing.T) { + var afters []string + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + after := r.URL.Query().Get("after") + afters = append(afters, after) + w.Header().Set("Content-Type", "application/json") + switch after { + case "": + fmt.Fprint(w, `{"data":[{"id":"eval_1","name":"dup"},{"id":"eval_2","name":"other"}],`+ + `"has_more":true,"last_id":"eval_2"}`) + default: + fmt.Fprint(w, `{"data":[{"id":"eval_3","name":"dup"}],"has_more":false}`) + } + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + require.Len(t, list.Data, 3, "both pages have to be gathered") + assert.Equal(t, []string{"", "eval_2"}, afters, + "the second request has to carry the cursor the first returned") + + var named int + for _, e := range list.Data { + if e.Name == "dup" { + named++ + } + } + assert.Equal(t, 2, named, "the duplicate on page two is what makes the name ambiguous") +} + +// The cursor is read only when the service sends one. Without this a service +// that omits it would loop forever or truncate, depending on the guard. +func TestListOpenAIEvalsStopsWithoutACursor(t *testing.T) { + calls := 0 + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"eval_1","name":"only"}]}`) + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, calls, "no cursor means one page, not an endless walk") +} + +// has_more with no last_id is the other way a service can leave the walk +// without an anchor, and repeating the same request would never terminate. +func TestListOpenAIEvalsStopsWhenTheCursorIsEmpty(t *testing.T) { + calls := 0 + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"eval_1"}],"has_more":true,"last_id":""}`) + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, calls, "has_more without last_id has nowhere to go") +} + +// An explicit limit still bounds the walk, and asks each page for only what is +// left rather than the whole limit again. +func TestListOpenAIEvalRunsHonoursTheLimit(t *testing.T) { + var limits []string + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + limits = append(limits, r.URL.Query().Get("limit")) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"run_1"},{"id":"run_2"}],"has_more":true,"last_id":"run_2"}`) + }) + + list, err := c.ListOpenAIEvalRuns(context.Background(), "eval_1", 2) + + require.NoError(t, err) + assert.Len(t, list.Data, 2, "the limit stops the walk") + assert.Equal(t, []string{"2"}, limits, "one page satisfied it") +} + +// run list without a limit has to report every run, not the newest page of +// them, or a run a caller started is missing from the list that should show it. +func TestListOpenAIEvalRunsGathersEveryPage(t *testing.T) { + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Query().Get("after") { + case "": + fmt.Fprint(w, `{"data":[{"id":"run_1"},{"id":"run_2"}],"has_more":true,"last_id":"run_2"}`) + case "run_2": + fmt.Fprint(w, `{"data":[{"id":"run_3"}],"has_more":true,"last_id":"run_3"}`) + default: + fmt.Fprint(w, `{"data":[{"id":"run_4"}],"has_more":false}`) + } + }) + + list, err := c.ListOpenAIEvalRuns(context.Background(), "eval_1", 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 4, "three pages, every run") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go new file mode 100644 index 00000000000..f1ad0912687 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// clientAndServer is clientServing plus the server, for tests that need to +// build an absolute link back to it. +func clientAndServer(t *testing.T, handler http.HandlerFunc) (*EvalClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewEvalClientFromPipeline(srv.URL, pipeline), srv +} + +// A nextLink is allowed to be relative, and a relative one has no host or +// scheme of its own. Comparing it to the endpoint before resolving refused a +// legitimate link, which turned a working listing into a hard failure. +func TestListEvaluatorVersionsFollowsARelativeNextLink(t *testing.T) { + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"/evaluators/e/versions?page=2"}`) + return + } + fmt.Fprint(w, `{"value":[{"name":"two"}]}`) + }) + + list, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + require.NoError(t, err, "a relative nextLink must be followed, not refused") + require.NotNil(t, list) + require.Len(t, list.Value, 2, "the second page has to be merged in") +} + +// Resolving relative links must not become a way to reach another host: a +// protocol-relative link keeps the scheme and swaps the host, and this client +// sends an Authorization header. +func TestListEvaluatorVersionsRefusesALinkResolvingToAnotherHost(t *testing.T) { + var elsewhereHits int32 + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&elsewhereHits, 1) + fmt.Fprint(w, `{"value":[{"name":"leaked"}]}`) + })) + t.Cleanup(elsewhere.Close) + + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"value":[{"name":"one"}],"nextLink":"//%s/evaluators"}`, + elsewhere.Listener.Addr().String()) + }) + + _, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + + require.Error(t, err, "a link resolving to another host must be refused") + assert.Zero(t, atomic.LoadInt32(&elsewhereHits), + "the token must never be sent to the other host") +} + +// A nextLink comes from the service, so a link that will not parse is the +// service's fault, not the caller's. Reporting it as an invalid endpoint sent +// the reader to check configuration that was never wrong. +func TestListEvaluatorVersionsBlamesTheLinkNotTheEndpoint(t *testing.T) { + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"https://host\u007f/x"}`) + }) + + _, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + + require.Error(t, err, "a nextLink the parser refuses has to fail") + assert.Contains(t, err.Error(), "nextLink", + "the reader has to know the service sent a bad link") + assert.NotContains(t, err.Error(), "invalid endpoint URL", + "their endpoint is fine and sending them to it wastes the investigation") +} + +// A cycle longer than one hop used to run to maxPages, because only a link +// pointing at the page it came from ended the walk. +func TestListEvaluatorVersionsStopsOnATwoPageCycle(t *testing.T) { + var hits int32 + var base string + c, srv := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + // a -> b -> a, so no link ever repeats the one immediately before it. + if r.URL.Query().Get("page") == "b" { + fmt.Fprintf(w, `{"value":[{"name":"b"}],"nextLink":%q}`, base+"/evaluators?page=a") + return + } + fmt.Fprintf(w, `{"value":[{"name":"a"}],"nextLink":%q}`, base+"/evaluators?page=b") + }) + base = srv.URL + + list, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + require.NoError(t, err, "a cycle ends the walk rather than failing the command") + require.NotNil(t, list) + assert.LessOrEqual(t, atomic.LoadInt32(&hits), int32(4), + "a two-page cycle must stop quickly, not run to maxPages") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go new file mode 100644 index 00000000000..454a574c496 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "log" + "strings" + "time" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// JobStatus — typed status with terminal/failed semantics +// --------------------------------------------------------------------------- + +// JobStatus represents the normalized status of a generation job. +type JobStatus string + +const ( + JobStatusRunning JobStatus = "running" + JobStatusCompleted JobStatus = "completed" + JobStatusSucceeded JobStatus = "succeeded" + JobStatusFailed JobStatus = "failed" + JobStatusCancelled JobStatus = "cancelled" + JobStatusCanceled JobStatus = "canceled" +) + +// ParseJobStatus normalizes a raw status string into a JobStatus. +// An empty string is treated as "running". +func ParseJobStatus(s string) JobStatus { + if s == "" { + return JobStatusRunning + } + return JobStatus(strings.ToLower(s)) +} + +// IsTerminal returns true when the status represents a final state. +func (s JobStatus) IsTerminal() bool { + switch s { + case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// IsFailed returns true when the status represents a failure or cancellation. +func (s JobStatus) IsFailed() bool { + switch s { + case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// String returns the status as a plain string. +func (s JobStatus) String() string { + return string(s) +} + +// --------------------------------------------------------------------------- +// JobFailedError — returned when a polled job reaches a failed state +// --------------------------------------------------------------------------- + +// JobFailedError is returned when a generation job reaches a failed terminal state. +type JobFailedError struct { + Job *GenerationJob + Status JobStatus +} + +func (e *JobFailedError) Error() string { + if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { + return messages.JobFailedWithReason(string(e.Status), e.Job.Error.Message) + } + return messages.JobFailed(string(e.Status)) +} + +// --------------------------------------------------------------------------- +// PollerTimeoutError — returned when polling exhausts all attempts +// --------------------------------------------------------------------------- + +// PollerTimeoutError is returned when a generation job has not reached a +// terminal state within the configured number of polling attempts. +type PollerTimeoutError struct { + OperationID string + Attempts int +} + +func (e *PollerTimeoutError) Error() string { + return messages.PollerTimedOut(e.OperationID, e.Attempts) +} + +// --------------------------------------------------------------------------- +// GetJobFunc — callback type for fetching job state +// --------------------------------------------------------------------------- + +// GetJobFunc fetches the current state of a generation job by operation ID. +type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) + +// --------------------------------------------------------------------------- +// PollerOptions — configurable polling behavior +// --------------------------------------------------------------------------- + +// PollerOptions configures the polling interval and attempt limit. +type PollerOptions struct { + Interval time.Duration + MaxAttempts int +} + +// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). +func DefaultPollerOptions() PollerOptions { + return PollerOptions{ + Interval: 2 * time.Second, + MaxAttempts: 300, + } +} + +// --------------------------------------------------------------------------- +// Poller — polls a generation job until it reaches a terminal state +// --------------------------------------------------------------------------- + +// Poller polls a GenerationJob until it reaches a terminal status. +type Poller struct { + OperationID string + APIVersion string + GetJob GetJobFunc + Options PollerOptions + // OnPoll is called after each successful poll with the latest status. + // Callers can use this for progress reporting (e.g. debug logging). + OnPoll func(status JobStatus) +} + +// NewPoller creates a Poller with default options. +func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { + return &Poller{ + OperationID: operationID, + APIVersion: apiVersion, + GetJob: getJob, + Options: DefaultPollerOptions(), + } +} + +// Poll blocks until the job reaches a terminal state, the context is +// cancelled, or the maximum number of attempts is exhausted. +// +// On success it returns the completed GenerationJob. +// On failure it returns a *JobFailedError (which wraps the job for inspection). +// On timeout it returns a plain error. +func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { + if p.OperationID == "" { + return nil, messages.OperationIDEmpty() + } + + for range p.Options.MaxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(p.Options.Interval): + } + + job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) + if err != nil { + if evalcore.IsTransientError(err) { + log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) + continue + } + return nil, err + } + + status := ParseJobStatus(job.Status) + log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) + + if p.OnPoll != nil { + p.OnPoll(status) + } + + if status.IsTerminal() { + if status.IsFailed() { + return nil, &JobFailedError{Job: job, Status: status} + } + return job, nil + } + } + + return nil, &PollerTimeoutError{ + OperationID: p.OperationID, + Attempts: p.Options.MaxAttempts, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go new file mode 100644 index 00000000000..55d02d27cd6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fastPoller polls without the two-second wait a real one takes, so a test +// covering an exhausted attempt budget finishes in milliseconds. +func fastPoller(t *testing.T, get GetJobFunc, maxAttempts int) *Poller { + t.Helper() + p := NewPoller("op_1", "v1", get) + p.Options = PollerOptions{Interval: time.Millisecond, MaxAttempts: maxAttempts} + return p +} + +// jobs answers each poll with the next status in turn, holding the last one. +func jobs(statuses ...string) (GetJobFunc, *atomic.Int32) { + var calls atomic.Int32 + return func(context.Context, string, string) (*GenerationJob, error) { + i := int(calls.Add(1)) - 1 + if i >= len(statuses) { + i = len(statuses) - 1 + } + return &GenerationJob{ID: "op_1", Status: statuses[i]}, nil + }, &calls +} + +// The service spells terminal states several ways and does not agree with +// itself on case. Missing one leaves the CLI polling a job that will never +// change again, until the attempt budget runs out minutes later. +func TestJobStatusTerminalAndFailed(t *testing.T) { + cases := []struct { + raw string + terminal bool + failed bool + }{ + {"running", false, false}, + {"queued", false, false}, + {"", false, false}, // an absent status is a job that has not started + {"completed", true, false}, + {"succeeded", true, false}, + {"Succeeded", true, false}, // the service does not agree with itself on case + {"COMPLETED", true, false}, + {"failed", true, true}, + {"cancelled", true, true}, // both spellings are in use + {"canceled", true, true}, + {"Cancelled", true, true}, + } + + for _, tc := range cases { + status := ParseJobStatus(tc.raw) + assert.Equal(t, tc.terminal, status.IsTerminal(), "IsTerminal(%q)", tc.raw) + assert.Equal(t, tc.failed, status.IsFailed(), "IsFailed(%q)", tc.raw) + } + + assert.Equal(t, JobStatusRunning, ParseJobStatus(""), + "a job with no status yet is running, not finished") + assert.Equal(t, "succeeded", ParseJobStatus("Succeeded").String()) +} + +// A job that reaches a success state is returned, and polling stops there +// rather than continuing to spend attempts. +func TestPollerReturnsOnSuccess(t *testing.T) { + get, calls := jobs("running", "running", "completed") + job, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.NoError(t, err) + require.NotNil(t, job) + assert.Equal(t, "completed", job.Status) + assert.Equal(t, int32(3), calls.Load(), "polling stops at the terminal state") +} + +// A failure has to carry the service's own message, since "job failed" alone +// leaves the user with nothing to act on. +func TestPollerReportsTheFailureMessage(t *testing.T) { + get := func(context.Context, string, string) (*GenerationJob, error) { + return &GenerationJob{ + ID: "op_1", + Status: "failed", + Error: &JobError{Message: "the model deployment was not found"}, + }, nil + } + + _, err := fastPoller(t, get, 5).Poll(context.Background()) + + var failed *JobFailedError + require.ErrorAs(t, err, &failed, "the caller inspects the job, so the type has to survive") + assert.Equal(t, JobStatusFailed, failed.Status) + assert.Contains(t, err.Error(), "the model deployment was not found") +} + +// A cancelled job is finished but not successful, so it must not be returned +// as a job whose output can be read. +func TestPollerTreatsCancellationAsFailure(t *testing.T) { + get, _ := jobs("cancelled") + job, err := fastPoller(t, get, 5).Poll(context.Background()) + + require.Error(t, err) + assert.Nil(t, job) + var failed *JobFailedError + require.ErrorAs(t, err, &failed) + assert.Equal(t, JobStatusCancelled, failed.Status) +} + +// A job with no error object still has to say something, or the failure +// surfaces as an empty line. +func TestJobFailedErrorWithoutAMessage(t *testing.T) { + err := &JobFailedError{Status: JobStatusFailed} + assert.Contains(t, err.Error(), "failed") +} + +// Throttling and server faults are the service being busy, not the job being +// broken: giving up on one would fail a run that was going to succeed. +func TestPollerRetriesTransientErrors(t *testing.T) { + var calls atomic.Int32 + get := func(context.Context, string, string) (*GenerationJob, error) { + switch calls.Add(1) { + case 1: + return nil, &azcore.ResponseError{StatusCode: http.StatusTooManyRequests} + case 2: + return nil, &azcore.ResponseError{StatusCode: http.StatusBadGateway} + case 3: + return nil, errors.New("read tcp: connection reset by peer") + default: + return &GenerationJob{ID: "op_1", Status: "succeeded"}, nil + } + } + + job, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.NoError(t, err) + assert.Equal(t, "succeeded", job.Status) + assert.Equal(t, int32(4), calls.Load()) +} + +// A real failure — a deleted job, a bad token — is not worth retrying for the +// rest of the budget, so it is returned at once. +func TestPollerStopsOnAPermanentError(t *testing.T) { + var calls atomic.Int32 + get := func(context.Context, string, string) (*GenerationJob, error) { + calls.Add(1) + return nil, &azcore.ResponseError{StatusCode: http.StatusNotFound} + } + + _, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.Error(t, err) + assert.Equal(t, int32(1), calls.Load(), "a 404 will not become a 200") +} + +// A job that never finishes has to end as a timeout naming the operation, so +// the user can go look it up rather than being told nothing. +func TestPollerTimesOut(t *testing.T) { + get, calls := jobs("running") + _, err := fastPoller(t, get, 3).Poll(context.Background()) + + var timeout *PollerTimeoutError + require.ErrorAs(t, err, &timeout) + assert.Equal(t, "op_1", timeout.OperationID) + assert.Equal(t, 3, timeout.Attempts) + assert.Contains(t, err.Error(), "op_1") + assert.Equal(t, int32(3), calls.Load(), "the budget is attempts, not retries after the first") +} + +// Ctrl-C during a wait has to return promptly rather than after the interval, +// and the context's own error is what says why. +func TestPollerHonoursContextCancellation(t *testing.T) { + get, _ := jobs("running") + p := NewPoller("op_1", "v1", get) + p.Options = PollerOptions{Interval: time.Hour, MaxAttempts: 10} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { _, err := p.Poll(ctx); done <- err }() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("Poll ignored a cancelled context and waited out the interval") + } +} + +// Progress reporting is what the user sees during a long run, so the callback +// fires on every poll and not only the last. +func TestPollerReportsEachStatus(t *testing.T) { + get, _ := jobs("queued", "running", "succeeded") + p := fastPoller(t, get, 10) + + var seen []JobStatus + p.OnPoll = func(s JobStatus) { seen = append(seen, s) } + + _, err := p.Poll(context.Background()) + require.NoError(t, err) + assert.Equal(t, []JobStatus{"queued", "running", "succeeded"}, seen) +} + +// Polling an empty id would ask the service about nothing, forever. It is the +// caller's bug and is worth naming before the first request. +func TestPollerRejectsAnEmptyOperationID(t *testing.T) { + _, err := NewPoller("", "v1", nil).Poll(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "operation ID") +} + +// The defaults bound a run at roughly ten minutes. A shorter budget would +// abandon generation jobs that legitimately take that long. +func TestDefaultPollerOptions(t *testing.T) { + opts := DefaultPollerOptions() + assert.Equal(t, 2*time.Second, opts.Interval) + assert.Equal(t, 300, opts.MaxAttempts) + assert.GreaterOrEqual(t, opts.Interval*time.Duration(opts.MaxAttempts), 10*time.Minute) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go new file mode 100644 index 00000000000..f4b642ee524 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSub = "/subscriptions/00000000-0000-0000-0000-000000000000" + +// Only a Foundry project builds a portal prefix. +// +// The check was "has a parent and a slash in its type", which every nested +// resource satisfies: a storage blob container reached this far and produced a +// confident link to a portal page that cannot exist. +func TestOnlyAFoundryProjectBuildsAPortalPrefix(t *testing.T) { + project := testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj" + _, err := NewPortalPrefix(project) + require.NoError(t, err, "a Foundry project is the thing this is for") + + notProjects := map[string]string{ + "a storage container": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.Storage/storageAccounts/sa/blobServices/default", + "the parent account": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct", + "a different child of the account": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/deployments/dep", + } + for what, id := range notProjects { + _, err := NewPortalPrefix(id) + assert.Errorf(t, err, "%s is not a Foundry project", what) + } +} + +// Names reach these builders from the service, not from this extension's own +// validation, so a space or a slash would otherwise produce a link that breaks +// when pasted or points at a different route. +func TestPortalURLsEscapeTheNamesTheyCarry(t *testing.T) { + prefix, err := NewPortalPrefix(testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + got := prefix.DatasetURL("my set", "1.0") + assert.NotContains(t, got, "my set", "a raw space does not survive a paste") + assert.Contains(t, got, "my%20set") + + got = prefix.EvaluatorURL("a/b", "1") + assert.Contains(t, got, "a%2Fb", "a slash would otherwise change the route") + assert.Equal(t, 1, strings.Count(got, "/build/evaluations/catalog/")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go new file mode 100644 index 00000000000..2612c296eaa --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" + + "azureaieval/internal/messages" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/google/uuid" +) + +// foundryProjectResourceType is the only resource a portal prefix can be built +// from. Every nested resource has a parent and a slash in its type, so matching +// on shape rather than on this let unrelated children through. +const foundryProjectResourceType = "Microsoft.CognitiveServices/accounts/projects" + +// PortalPrefix holds the parsed project context needed to construct Foundry portal URLs. +type PortalPrefix struct { + prefix string // e.g. "https://ai.azure.com/nextgen/r/,,,," +} + +// NewPortalPrefix parses an ARM project resource ID and returns a PortalPrefix +// that can be reused to build multiple portal URLs. +// Returns an error if the resource ID is invalid or not a Foundry project. +func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { + resourceID, err := arm.ParseResourceID(projectResourceID) + if err != nil { + return nil, messages.ParsingProjectResourceID(err) + } + + encodedSub, err := encodeSubscriptionForURL(resourceID.SubscriptionID) + if err != nil { + return nil, messages.EncodingSubscriptionID(err) + } + + // The exact type, not merely a nested one. Any child resource has a parent + // and a slash in its type -- a storage container reached this far and built a + // plausible URL onto a portal page that does not exist. + if resourceID.Parent == nil || + !strings.EqualFold(resourceID.ResourceType.String(), foundryProjectResourceType) { + return nil, messages.NotAFoundryProjectResourceID(projectResourceID) + } + + prefix := fmt.Sprintf( + "https://ai.azure.com/nextgen/r/%s,%s,,%s,%s", + encodedSub, resourceID.ResourceGroupName, + resourceID.Parent.Name, resourceID.Name, + ) + return &PortalPrefix{prefix: prefix}, nil +} + +// EvalRunURL returns the portal URL for an eval run report. +func (p *PortalPrefix) EvalRunURL(evalID, runID string) string { + return fmt.Sprintf("%s/build/evaluations/%s/run/%s", + p.prefix, url.PathEscape(evalID), url.PathEscape(runID)) +} + +// EvaluatorURL returns the portal URL for a generated evaluator. +func (p *PortalPrefix) EvaluatorURL(evaluatorName, version string) string { + return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", + p.prefix, url.PathEscape(evaluatorName), url.PathEscape(version)) +} + +// DatasetURL returns the portal URL for a dataset. +// +// Escaped rather than interpolated: these names are the service's, not this +// extension's, so a space or a slash in one would otherwise produce a link that +// breaks when pasted or points somewhere else entirely. +func (p *PortalPrefix) DatasetURL(datasetName, version string) string { + return fmt.Sprintf("%s/build/data/datasets/%s/%s", + p.prefix, url.PathEscape(datasetName), url.PathEscape(version)) +} + +// OptimizationURL returns the portal URL for an optimization job. +func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { + return fmt.Sprintf("%s/build/agents/%s/optimization/%s", + p.prefix, url.PathEscape(agentName), url.PathEscape(operationID)) +} + +// encodeSubscriptionForURL encodes a subscription ID GUID as base64 without padding. +func encodeSubscriptionForURL(subscriptionID string) (string, error) { + guid, err := uuid.Parse(subscriptionID) + if err != nil { + return "", messages.InvalidSubscriptionID(err) + } + guidBytes, _ := guid.MarshalBinary() + return strings.TrimRight(base64.URLEncoding.EncodeToString(guidBytes), "="), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go new file mode 100644 index 00000000000..971331cb9b5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testProjectID = "/subscriptions/00000000-1111-2222-3333-444444444444/" + + "resourceGroups/rg-eval/providers/Microsoft.CognitiveServices/accounts/acct/projects/proj" + +// A portal URL is printed at the end of a run and is the one thing a user +// clicks. It is assembled from parts rather than returned by the service, so +// nothing but a test says whether it lands anywhere. +func TestPortalPrefix_BuildsEveryDocumentedURL(t *testing.T) { + p, err := NewPortalPrefix(testProjectID) + require.NoError(t, err) + + // The subscription travels base64url-encoded without padding, so the + // literal GUID must not appear anywhere in the result. + const sub = "00000000-1111-2222-3333-444444444444" + + tests := []struct { + name string + got string + want string + }{ + {"eval run", p.EvalRunURL("eval_1", "evalrun_1"), "/build/evaluations/eval_1/run/evalrun_1"}, + {"evaluator", p.EvaluatorURL("quality", "3"), "/build/evaluations/catalog/quality/3"}, + {"dataset", p.DatasetURL("regression", "2"), "/build/data/datasets/regression/2"}, + {"optimization", p.OptimizationURL("support", "op_9"), "/build/agents/support/optimization/op_9"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, strings.HasPrefix(tt.got, "https://ai.azure.com/nextgen/r/"), + "got %s", tt.got) + assert.True(t, strings.HasSuffix(tt.got, tt.want), "got %s", tt.got) + assert.Contains(t, tt.got, "rg-eval") + assert.Contains(t, tt.got, "acct") + assert.Contains(t, tt.got, "proj") + assert.NotContains(t, tt.got, sub, + "the subscription is encoded, so its plain GUID must not appear") + }) + } +} + +// The encoding is what the portal decodes on the other end, so it is pinned +// rather than merely exercised. +func TestEncodeSubscriptionForURL(t *testing.T) { + encoded, err := encodeSubscriptionForURL("00000000-1111-2222-3333-444444444444") + + require.NoError(t, err) + assert.NotContains(t, encoded, "=", "padding would need escaping inside a URL segment") + assert.NotContains(t, encoded, "+", "base64url, not standard base64") + assert.NotContains(t, encoded, "/", "a slash would split the URL segment") + assert.Equal(t, "AAAAABERIiIzM0RERERERA", encoded) +} + +func TestEncodeSubscriptionForURL_RejectsSomethingThatIsNotAGUID(t *testing.T) { + _, err := encodeSubscriptionForURL("not-a-subscription") + + require.Error(t, err) + assert.Contains(t, err.Error(), "subscription") +} + +// A resource ID that is not a project has no account to name, and guessing +// would produce a URL that resolves to someone else's project. +func TestNewPortalPrefix_RefusesWhatIsNotAProject(t *testing.T) { + tests := []struct { + name string + id string + }{ + {"not a resource id at all", "hello"}, + {"empty", ""}, + { + name: "an account rather than a project under it", + id: "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct", + }, + { + name: "a project whose subscription is not a GUID", + id: "/subscriptions/not-a-guid/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := NewPortalPrefix(tt.id) + + require.Error(t, err) + assert.Nil(t, p) + }) + } +} + +// The prefix distinguishes built-in evaluators from ones the project owns, +// which is what decides whether a version is published or referenced. +func TestIsBuiltinEvaluator(t *testing.T) { + assert.True(t, IsBuiltinEvaluator("builtin.task_adherence")) + assert.False(t, IsBuiltinEvaluator("task_adherence")) + assert.False(t, IsBuiltinEvaluator("builtin"), "the dot is part of the prefix") + assert.False(t, IsBuiltinEvaluator("my.builtin.thing"), "the prefix has to lead") + assert.False(t, IsBuiltinEvaluator("")) +} + +func TestSplitEvaluators(t *testing.T) { + // `- evaluator: builtin.coherence` fills Evaluator and leaves Name empty. + // Name is the criterion label in results, so a fixture that put the + // reference there agreed with the bug rather than with any real config. + generated, builtin := SplitEvaluators(evalcore.EvaluatorList{ + {Evaluator: "builtin.coherence"}, + {Evaluator: "support-quality"}, + {Evaluator: "builtin.task_adherence", Name: "adherence"}, + }) + + require.Len(t, generated, 1) + assert.Equal(t, "support-quality", generated[0].Evaluator) + require.Len(t, builtin, 2) + assert.Equal(t, "builtin.coherence", builtin[0].Evaluator) + assert.Equal(t, "builtin.task_adherence", builtin[1].Evaluator, + "a criterion label does not stop a built-in being a built-in") +} + +// Both halves come back nil rather than empty for an empty input, so a caller +// checking len() reads the same either way. +func TestSplitEvaluators_Empty(t *testing.T) { + generated, builtin := SplitEvaluators(nil) + + assert.Empty(t, generated) + assert.Empty(t, builtin) +} + +// This decides whether a value is looked up in the service or opened off disk. +// Getting it wrong sends a path to the registry, or a registered name to the +// filesystem, and neither failure names the real problem. +func TestIsDatasetName(t *testing.T) { + names := []string{ + "support-regression", + "dataset_v2", + "name.with.dots", + "trailing.txt", + } + for _, v := range names { + assert.Truef(t, IsDatasetName(v), "%q is a registered name", v) + } + + paths := []string{ + "", + "data.jsonl", + "data.json", + "data.csv", + "DATA.JSONL", + "./data.jsonl", + "evals/datasets/x.jsonl", + `evals\datasets\x.jsonl`, + "a/b", + } + for _, v := range paths { + assert.Falsef(t, IsDatasetName(v), "%q is a path, not a name", v) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go new file mode 100644 index 00000000000..9f319fc586c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newRecordingClient points a client at a test server, with no credential +// policy in the pipeline. +// +// MaxRetries -1 disables the SDK's retry policy, so a test that answers 5xx on +// purpose does not spend ten seconds being retried. +func newRecordingClient(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return NewEvalClientFromPipeline(server.URL, runtime.NewPipeline( + "test", "v1.0.0", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}})) +} + +// versionServer answers a version listing and a publish, assigning whatever +// version the caller decides for each attempt. +func versionServer(t *testing.T, existing []string, assign func(attempt int) string) ( + http.HandlerFunc, *atomic.Int32, +) { + t.Helper() + var publishes atomic.Int32 + + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == http.MethodGet { + values := []map[string]any{} + for _, v := range existing { + values = append(values, map[string]any{"name": "tone", "version": v}) + } + if len(existing) == 0 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"NotFound"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + return + } + + attempt := int(publishes.Add(1)) + w.WriteHeader(http.StatusCreated) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "tone", "version": assign(attempt), + })) + }, &publishes +} + +// A name the project has never seen has no version to collide with, so it must +// publish once and return. Waiting there would tax every first publish for a +// hazard that cannot apply. +func TestCreateEvaluatorVersion_FirstPublishDoesNotRetry(t *testing.T) { + handler, publishes := versionServer(t, nil, func(int) string { return "1" }) + client := newRecordingClient(t, handler) + + started := time.Now() + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "1", created.Version) + assert.Equal(t, int32(1), publishes.Load(), "a first publish must be issued once") + assert.Less(t, time.Since(started), versionSettleInterval, + "a first publish must not wait on a version that cannot exist") +} + +// For a few seconds after a publish the service answers the next one with the +// version it just assigned, replacing that version rather than adding one. +// Accepting it would leave every eval bound to the earlier version scoring +// against a definition nobody chose, so the publish is reissued until the +// version advances. +func TestCreateEvaluatorVersion_RetriesUntilTheVersionAdvances(t *testing.T) { + handler, publishes := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 3 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(3), publishes.Load(), + "the publish must be reissued until the service assigns a new version") +} + +// A service that never advances must end in an error rather than in a version +// the caller believes is new. Reporting success there is the failure the whole +// guard exists to prevent. +func TestCreateEvaluatorVersion_GivesUpRatherThanReportASharedVersion(t *testing.T) { + handler, _ := versionServer(t, []string{"4"}, func(int) string { return "4" }) + client := newRecordingClient(t, handler) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := client.CreateEvaluatorVersion( + ctx, "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.Error(t, err) +} + +// The version listing lags a publish: asked immediately after a create it +// answers 404. A guard that trusted it would stand down in exactly the window +// it exists for, which is why the caller supplies the version it has already +// read. +func TestCreateEvaluatorVersion_UsesTheCallersVersionWhenTheListingLags(t *testing.T) { + handler, publishes := versionServer(t, nil, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), json.RawMessage(`{"version":"1"}`), "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(2), publishes.Load(), + "the version the caller read must be enough to catch the collision") +} + +// A version the service does not number cannot be compared, so it is taken at +// face value: refusing it would make an evaluator unpublishable over a +// convention this extension does not own. +func TestParseVersionNumber(t *testing.T) { + assert.Equal(t, 7, parseVersionNumber("7")) + assert.Equal(t, 0, parseVersionNumber("v7")) + assert.Equal(t, 0, parseVersionNumber("")) +} + +// The publish is reissued, so the same body has to arrive every time. A +// closure that consumed its body on the first attempt would send an empty one +// on the second and publish an evaluator with no definition. +func TestCreateEvaluatorVersion_ReissuesTheSameBody(t *testing.T) { + bodies := make(chan string, 4) + handler, _ := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + buf := make([]byte, r.ContentLength) + _, _ = r.Body.Read(buf) + bodies <- string(buf) + } + handler(w, r) + }) + + _, err := client.CreateEvaluatorVersion( + context.Background(), "tone", + json.RawMessage(`{"definition":{"type":"rubric"}}`), nil, "2025-11-15-preview") + require.NoError(t, err) + close(bodies) + + seen := 0 + for body := range bodies { + seen++ + assert.Contains(t, body, "rubric", fmt.Sprintf("attempt %d sent an empty body", seen)) + } + assert.Equal(t, 2, seen) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go new file mode 100644 index 00000000000..8f572baa0d8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service reads the window and the cap from inside trace_source and +// ignores them beside the data source type. Verified live: a run submitted with +// them at the top level came back with the nested ones null, so the caller's +// window was dropped without a word -- the same defect the legacy +// azure_ai_traces shape has, moved one level in. +func TestTracePreviewNestsEverythingInsideTheFilter(t *testing.T) { + ds := NewTracePreviewDataSource( + "support-agent", "2", + time.Unix(1785542400, 0), time.Unix(1785628800, 0), + 20, + ) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(raw, &got)) + + assert.Equal(t, "azure_ai_trace_data_source_preview", got["type"]) + assert.NotContains(t, got, "start_time", "the service ignores it here") + assert.NotContains(t, got, "end_time", "the service ignores it here") + assert.NotContains(t, got, "max_traces", "the service ignores it here") + + filter, ok := got["trace_source"].(map[string]any) + require.True(t, ok, "trace_source has to be an object") + assert.Equal(t, "agent_filter", filter["type"]) + assert.Equal(t, "support-agent", filter["agent_name"]) + assert.Equal(t, "2", filter["agent_version"]) + assert.EqualValues(t, 1785542400, filter["start_time"]) + assert.EqualValues(t, 1785628800, filter["end_time"]) + assert.EqualValues(t, 20, filter["max_traces"]) +} + +// An unpinned version and an open window are omitted rather than sent as zero, +// which the service would read as a bound of 1970. +func TestTracePreviewOmitsWhatWasNotAskedFor(t *testing.T) { + ds := NewTracePreviewDataSource("support-agent", "", time.Time{}, time.Time{}, 0) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(raw, &got)) + + filter, ok := got["trace_source"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "support-agent", filter["agent_name"]) + assert.NotContains(t, filter, "agent_version") + assert.NotContains(t, filter, "start_time") + assert.NotContains(t, filter, "end_time") + assert.NotContains(t, filter, "max_traces") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go new file mode 100644 index 00000000000..d9bd6715a6c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "azureaieval/internal/messages" + + "go.yaml.in/yaml/v3" +) + +// BuiltinPrefix marks an evaluator provided by the platform. The prefix is +// stripped before the name is sent as testing_criteria[].evaluator_name. +const BuiltinPrefix = "builtin." + +// EvaluatorRef is one entry in an eval's `evaluators:` list. Every entry is a +// map keyed `evaluator:`; what to publish lives on the catalog entry instead, +// so a reference only names an evaluator and says how to run it: +// +// evaluators: +// - evaluator: builtin.task_adherence +// initialization_parameters: +// model: gpt-5.6-luna +// threshold: 3 +// - evaluator: support-agent-quality +// name: quality_strict +// version: "2" +// data_mapping: +// query: "{{item.customer_message}}" +type EvaluatorRef struct { + // Evaluator is the evaluator to run: a catalog name or builtin.. + Evaluator string `yaml:"evaluator" json:"evaluator"` + // Name labels the criterion in results. Empty means the evaluator's name. + Name string `yaml:"name,omitempty" json:"name,omitempty"` + // Version pins this reference. Pinning belongs to one eval's reference + // rather than to the asset, matching evaluator_version on the criterion. + Version string `yaml:"version,omitempty" json:"version,omitempty"` + // InitializationParameters carry the judge deployment and a built-in's + // numeric threshold. They are bound against the evaluator's published + // contract rather than forwarded as written. + InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` + // DataMapping binds evaluator inputs to dataset columns, and is written + // only when the inference from declared inputs and columns gets it wrong. + DataMapping map[string]string `yaml:"data_mapping,omitempty" json:"data_mapping,omitempty"` +} + +// IsBuiltin reports whether the reference names a platform evaluator, which +// needs no catalog entry and is never uploaded. +func (e EvaluatorRef) IsBuiltin() bool { + return strings.HasPrefix(e.Evaluator, BuiltinPrefix) +} + +// APIName is the name the service expects, with the builtin prefix removed. +func (e EvaluatorRef) APIName() string { + return strings.TrimPrefix(e.Evaluator, BuiltinPrefix) +} + +// CriterionName labels this criterion in results. +func (e EvaluatorRef) CriterionName() string { + if e.Name != "" { + return e.Name + } + return e.APIName() +} + +// EvaluatorList is a sequence of EvaluatorRef. +// +// A bare string is refused rather than accepted quietly. Every other collection +// in the file is a list of named maps, and a bare string would have to mean the +// evaluator while reading as the criterion's own name — a different key this +// same entry also carries. +type EvaluatorList []EvaluatorRef + +func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.SequenceNode { + return messages.EvaluatorsMustBeSequence(value.Kind) + } + + result := make([]EvaluatorRef, 0, len(value.Content)) + for _, node := range value.Content { + switch node.Kind { + case yaml.ScalarNode: + var name string + if err := node.Decode(&name); err != nil { + return messages.DecodingEvaluatorName(err) + } + return messages.BareEvaluatorEntry(name) + case yaml.MappingNode: + ref, err := decodeEvaluatorRef(node) + if err != nil { + return err + } + if ref.Evaluator == "" { + return messages.EvaluatorEntryMissingEvaluator() + } + result = append(result, ref) + default: + return messages.EvaluatorEntryMustBeMapping(node.Kind) + } + } + + *el = result + return nil +} + +// decodeEvaluatorRef decodes one entry with the strictness the file promises. +// +// yaml.Node.Decode does not inherit KnownFields from the decoder that reached +// it, so `verison:` inside an evaluator entry was dropped in silence while the +// same typo one level up was named. Round-tripping the node through a strict +// decoder restores it; the error keeps yaml's own "field X not found in type Y" +// shape, which the caller rewrites into the file's vocabulary. +func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { + raw, err := yaml.Marshal(node) + if err != nil { + return EvaluatorRef{}, messages.DecodingEvaluator(err) + } + + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + decoder.KnownFields(true) + + var ref EvaluatorRef + if err := decoder.Decode(&ref); err != nil { + return EvaluatorRef{}, messages.DecodingEvaluator(rebaseYAMLLines(err, node.Line)) + } + return ref, nil +} + +// yamlErrorLine matches the line number yaml puts on each unmarshal error. +var yamlErrorLine = regexp.MustCompile(`line (\d+):`) + +// rebaseYAMLLines moves line numbers from the extracted snippet back onto the +// file, so the reader is pointed at the key they typed rather than at line 2. +func rebaseYAMLLines(err error, startLine int) error { + if startLine <= 0 { + return err + } + return errors.New(yamlErrorLine.ReplaceAllStringFunc(err.Error(), func(m string) string { + n, convErr := strconv.Atoi(yamlErrorLine.FindStringSubmatch(m)[1]) + if convErr != nil { + return m + } + return fmt.Sprintf("line %d:", startLine+n-1) + })) +} + +// UnmarshalJSON accepts the same mapping-only form as the YAML decoder. +// +// This matters for the service-target provider: azd hands the service entry to +// the extension as JSON, so a config written the old way arrives here as a bare +// string and has to be refused with the same remedy rather than with a +// decoder's own type error. +func (el *EvaluatorList) UnmarshalJSON(data []byte) error { + var entries []json.RawMessage + if err := json.Unmarshal(data, &entries); err != nil { + return messages.EvaluatorsMustBeList(err) + } + + result := make([]EvaluatorRef, 0, len(entries)) + for _, entry := range entries { + trimmed := bytes.TrimSpace(entry) + if len(trimmed) > 0 && trimmed[0] == '"' { + var name string + if err := json.Unmarshal(trimmed, &name); err != nil { + return messages.DecodingEvaluatorName(err) + } + return messages.BareEvaluatorEntry(name) + } + + var ref EvaluatorRef + if err := json.Unmarshal(trimmed, &ref); err != nil { + return messages.DecodingEvaluator(err) + } + if ref.Evaluator == "" { + return messages.EvaluatorEntryMissingEvaluator() + } + result = append(result, ref) + } + + *el = result + return nil +} + +// MarshalJSON is the default list encoding, defined so a compact form cannot +// creep back in through the encoder. +// +// Everything the reference carries has to survive the round trip: the eval +// fingerprint is taken over this encoding, so a field dropped here is a change +// the reconciler cannot see. +func (el EvaluatorList) MarshalJSON() ([]byte, error) { + // Aliased so the element encoder does not recurse through this method. + type ref = EvaluatorRef + + out := make([]any, 0, len(el)) + for _, r := range el { + out = append(out, ref(r)) + } + return json.Marshal(out) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go new file mode 100644 index 00000000000..b185414e8ed --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// The service-target provider receives the config as JSON, not YAML, so both +// paths have to decode the mapping form identically. Supporting only YAML made +// `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesEntriesFromJSON(t *testing.T) { + const payload = `[ + {"evaluator": "builtin.task_adherence"}, + {"evaluator": "support-quality", "name": "quality_strict", + "initialization_parameters": {"model": "gpt-5.6-luna", "threshold": 4}}, + {"evaluator": "pinned", "version": "3"} + ]` + + var list EvaluatorList + require.NoError(t, json.Unmarshal([]byte(payload), &list)) + require.Len(t, list, 3) + + require.Equal(t, "builtin.task_adherence", list[0].Evaluator) + require.True(t, list[0].IsBuiltin()) + require.Equal(t, "task_adherence", list[0].APIName()) + require.Equal(t, "task_adherence", list[0].CriterionName()) + require.Nil(t, list[0].InitializationParameters) + + require.Equal(t, "support-quality", list[1].Evaluator) + require.Equal(t, "quality_strict", list[1].CriterionName()) + require.False(t, list[1].IsBuiltin()) + require.Equal(t, "gpt-5.6-luna", list[1].InitializationParameters["model"]) + require.EqualValues(t, 4, list[1].InitializationParameters["threshold"]) + + require.Equal(t, "pinned", list[2].Evaluator) + require.Equal(t, "3", list[2].Version) +} + +// The JSON and YAML decoders must agree, otherwise a config behaves one way +// through the CLI and another through `azd up`. +func TestEvaluatorListJSONMatchesYAML(t *testing.T) { + const doc = ` +- evaluator: builtin.task_adherence +- evaluator: support-quality + name: quality_strict + initialization_parameters: + model: gpt-5.6-luna + data_mapping: + query: "{{item.customer_message}}" +` + var fromYAML EvaluatorList + require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) + + encoded, err := json.Marshal(fromYAML) + require.NoError(t, err) + + var fromJSON EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &fromJSON)) + require.Equal(t, fromYAML, fromJSON) +} + +// A bare string is the old shorthand. It has to be refused with the remedy +// rather than a decoder type error, through both decoders, because the +// service-target provider only ever sees JSON. +func TestEvaluatorListRefusesBareString(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- builtin.task_adherence\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`["builtin.task_adherence"]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) +} + +func TestEvaluatorListRejectsEntryWithoutEvaluator(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- name: quality_strict\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"name": "quality_strict"}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) +} + +// The eval fingerprint is taken over this encoding, so a field the encoder +// drops is a change the reconciler cannot see. +func TestEvaluatorListMarshalKeepsEveryField(t *testing.T) { + list := EvaluatorList{{ + Evaluator: "support-quality", + Name: "quality_strict", + Version: "2", + InitializationParameters: map[string]any{"model": "gpt-5.6-luna"}, + DataMapping: map[string]string{"query": "{{item.customer_message}}"}, + }} + + encoded, err := json.Marshal(list) + require.NoError(t, err) + + var round EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &round)) + require.Equal(t, list, round) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go new file mode 100644 index 00000000000..029af83ffc0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "errors" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsTransientError reports whether err is worth retrying: throttling, a server +// fault, or a dropped connection. +func IsTransientError(err error) bool { + if err == nil { + return false + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 429 || respErr.StatusCode >= 500 + } + + msg := err.Error() + return strings.Contains(msg, "connection reset") || + strings.Contains(msg, "connection refused") || + strings.Contains(msg, "EOF") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go new file mode 100644 index 00000000000..8c2b397ccb4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "go.yaml.in/yaml/v3" +) + +// AgentHost is the service host the agents extension registers. Services +// declaring it are the ones that could be a generation target. +const AgentHost = "azure.ai.agent" + +// Where `azd ai agent optimize` leaves the configuration it settled on. +// +// These are the agents extension's file names, repeated rather than imported: +// azd extensions are separate Go modules and share no code, so the only way to +// read another one's output is to know its layout. That makes this a coupling +// worth naming — if the agents extension moves these, generation quietly stops +// finding instructions locally and falls back to the service. +const ( + agentConfigsDir = ".agent_configs" + agentBaselineDir = "baseline" + agentMetadataFile = "metadata.yaml" +) + +// agentConfigMetadata is the part of the optimize configuration's metadata.yaml +// that says where the instructions are. It points at a file rather than +// carrying the text, because the text is what a reviewer diffs. +type agentConfigMetadata struct { + InstructionFile string `yaml:"instruction_file"` +} + +// ErrAmbiguousAgentService reports that a target name matched more than one +// service, so there is no single set of instructions to read. +var ErrAmbiguousAgentService = messages.ErrAmbiguousAgentService + +// AgentInstructionsFromProject reads the target agent's instructions out of the +// project, returning empty when the project does not hold them. +// +// The instructions an agent was optimized with are the best description of what +// it is supposed to do, and they are already on disk, so generating from them +// needs no service call. Coming back empty is ordinary — most projects have +// never run `azd ai agent optimize` — and leaves the caller free to ask the +// service instead. +// +// The returned path is where the text came from, for a caller that wants to say +// so. +func AgentInstructionsFromProject( + proj *azdext.ProjectConfig, + agentName string, +) (instruction string, path string, err error) { + svc, err := findAgentService(proj, agentName) + if err != nil || svc == nil { + return "", "", err + } + + configDir := filepath.Join( + proj.GetPath(), serviceRelativeDir(svc), agentConfigsDir, agentBaselineDir) + + data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project + if err != nil { + // An agent that was never optimized has no such directory, which is + // the common case rather than a problem. + return "", "", nil + } + + var meta agentConfigMetadata + if err := yaml.Unmarshal(data, &meta); err != nil { + return "", "", messages.ReadingPath(filepath.Join(configDir, agentMetadataFile), err) + } + if meta.InstructionFile == "" { + return "", "", nil + } + + instructionPath := meta.InstructionFile + if !filepath.IsAbs(instructionPath) { + instructionPath = filepath.Join(configDir, instructionPath) + } + // The pointer comes out of the checkout, so it carries the checkout's + // trust. Left alone, an absolute path or one climbing out with `..` reads a + // file the project does not contain and sends it on as agent instructions. + if !withinDir(proj.GetPath(), instructionPath) { + return "", "", messages.InstructionFileOutsideProject( + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile) + } + text, err := os.ReadFile(instructionPath) //nolint:gosec // checked to be inside the project + if err != nil { + // The metadata named a file that is not there. That is worth saying: + // something wrote the pointer and not the target. + return "", "", messages.InstructionFileUnreadable( + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile, err) + } + + return strings.TrimSpace(string(text)), instructionPath, nil +} + +// withinDir reports whether path resolves to somewhere inside root. +// +// Compared after cleaning both, so `..` segments are resolved before the +// question is asked rather than matched as text. +func withinDir(root, path string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// findAgentService resolves a target name to the one service that is it. +// +// A name can match either the azure.yaml service key or the agent name the +// service declares, because the two need not agree and a user has only ever +// seen one of them. Matching both is what makes `--target` mean what they +// typed; refusing a tie is what stops it silently meaning one of two things. +func findAgentService( + proj *azdext.ProjectConfig, + agentName string, +) (*azdext.ServiceConfig, error) { + if proj == nil || agentName == "" { + return nil, nil + } + + var matched []string + services := map[string]*azdext.ServiceConfig{} + for name, svc := range proj.GetServices() { + if svc.GetHost() != AgentHost { + continue + } + if name == agentName || declaredAgentName(svc) == agentName { + matched = append(matched, name) + services[name] = svc + } + } + + switch len(matched) { + case 0: + return nil, nil + case 1: + return services[matched[0]], nil + default: + sort.Strings(matched) + return nil, messages.AmbiguousAgentService(agentName, matched) + } +} + +// declaredAgentName is the name the service gives the agent, which is what the +// service publishes under and so what the eval configuration's target refers +// to. It is absent when the service key is also the agent name. +func declaredAgentName(svc *azdext.ServiceConfig) string { + props := serviceProps(svc) + if props == nil { + return "" + } + name, _ := props.AsMap()["name"].(string) + return name +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go new file mode 100644 index 00000000000..dae36a18c5b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// writeOptimizeConfig lays out what `azd ai agent optimize` leaves behind: +// .agent_configs/baseline/metadata.yaml pointing at instructions.md beside it. +func writeOptimizeConfig(t *testing.T, serviceDir, metadata, instructions string) { + t.Helper() + dir := filepath.Join(serviceDir, ".agent_configs", "baseline") + require.NoError(t, os.MkdirAll(dir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.yaml"), []byte(metadata), 0o600)) + if instructions != "" { + require.NoError(t, + os.WriteFile(filepath.Join(dir, "instructions.md"), []byte(instructions), 0o600)) + } +} + +// agentService builds a project holding one agent service, optionally +// declaring an agent name that differs from the service key. +func agentService(t *testing.T, root, serviceKey, declaredName string) *azdext.ProjectConfig { + t.Helper() + svc := &azdext.ServiceConfig{ + Name: serviceKey, + Host: AgentHost, + RelativePath: serviceKey, + } + if declaredName != "" { + props, err := structpb.NewStruct(map[string]any{"name": declaredName}) + require.NoError(t, err) + svc.AdditionalProperties = props + } + return &azdext.ProjectConfig{ + Path: root, + Services: map[string]*azdext.ServiceConfig{serviceKey: svc}, + } +} + +// The instructions an agent was optimized with are already on disk, so +// generating from them needs no service call. +func TestAgentInstructionsFromProject_ReadsTheOptimizeConfig(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "support"), + "name: support\ninstruction_file: instructions.md\n", + "Answer support questions politely.\n") + + instruction, path, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + require.NoError(t, err) + assert.Equal(t, "Answer support questions politely.", instruction) + assert.Equal(t, filepath.Join(root, "support", ".agent_configs", "baseline", "instructions.md"), + path) +} + +// A target names the agent, which need not be spelled the way the azure.yaml +// key is. A user has only ever seen one of the two. +func TestAgentInstructionsFromProject_MatchesTheDeclaredAgentName(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "svc"), + "instruction_file: instructions.md\n", + "Be helpful.") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "svc", "support-agent"), "support-agent") + + require.NoError(t, err) + assert.Equal(t, "Be helpful.", instruction) +} + +// Most projects have never run optimize, so finding nothing is the ordinary +// case and has to leave the caller free to ask the service instead. +func TestAgentInstructionsFromProject_SilentWhenThereIsNothingToRead(t *testing.T) { + root := t.TempDir() + + tests := []struct { + name string + proj *azdext.ProjectConfig + agent string + }{ + {"no project at all", nil, "support"}, + {"no agent named", agentService(t, root, "support", ""), ""}, + {"no service by that name", agentService(t, root, "support", ""), "other"}, + {"no optimize config on disk", agentService(t, root, "support", ""), "support"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instruction, path, err := AgentInstructionsFromProject(tt.proj, tt.agent) + + assert.NoError(t, err) + assert.Empty(t, instruction) + assert.Empty(t, path) + }) + } +} + +// A service that is not an agent is not a candidate, however it is named. +func TestAgentInstructionsFromProject_IgnoresServicesThatAreNotAgents(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "Be helpful.") + + proj := agentService(t, root, "support", "") + proj.Services["support"].Host = "containerapp" + + instruction, _, err := AgentInstructionsFromProject(proj, "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +// Two services answering to one name is a tie, and picking either would make +// the generated dataset describe an agent the caller did not mean. +func TestAgentInstructionsFromProject_RefusesAnAmbiguousTarget(t *testing.T) { + root := t.TempDir() + proj := agentService(t, root, "support", "") + props, err := structpb.NewStruct(map[string]any{"name": "support"}) + require.NoError(t, err) + proj.Services["helpdesk"] = &azdext.ServiceConfig{ + Name: "helpdesk", Host: AgentHost, RelativePath: "helpdesk", + AdditionalProperties: props, + } + + _, _, err = AgentInstructionsFromProject(proj, "support") + + require.ErrorIs(t, err, ErrAmbiguousAgentService) + assert.Contains(t, err.Error(), "helpdesk") + assert.Contains(t, err.Error(), "support") + assert.Contains(t, err.Error(), "--target", + "an ambiguity the caller can resolve has to say how") +} + +// A pointer with nothing behind it means something wrote half the config. +// Falling back silently would generate from the published agent while the +// author believes they are generating from what they just optimized. +func TestAgentInstructionsFromProject_ReportsADanglingInstructionFile(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "instructions.md") +} + +// Metadata that names no instruction file is a config without instructions, +// not a broken one. +func TestAgentInstructionsFromProject_NoInstructionFileIsNotAnError(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "name: support\n", "") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +func TestAgentInstructionsFromProject_ReportsUnreadableMetadata(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "\tnot: [valid\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "metadata.yaml") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go new file mode 100644 index 00000000000..1a7277c390a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "strings" + + "azureaieval/internal/messages" +) + +// Conventional artifact locations, relative to the eval directory. +const ( + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` + // Version is what the generation job published. It is reported to the + // author rather than written into the catalog: an evaluator cannot carry + // both a `source:` and a `version:`, and pinning a generated dataset would + // freeze it against the very edit it exists to be the starting point for. + Version string `json:"version,omitempty"` +} + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// Sources a dataset can be generated from. +const ( + GenerateFromTraces = "traces" + GenerateFromAgent = "agent" + GenerateFromPrompt = "prompt" + GenerateFromFile = "file" +) + +// GenerateSources is what --from accepts, in help order. +var GenerateSources = []string{ + GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile, +} + +// ValidateGenerateSource rejects a --from value the service has no path for. +func ValidateGenerateSource(from string) error { + switch from { + case "", GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile: + return nil + default: + return messages.FromNotASource(from, GenerateSources) + } +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return messages.SampleSizeOutOfRange(MinSampleSize, MaxSampleSize, n) + } + return nil +} + +// ArtifactPath resolves an output directory against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := outputDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(outputDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognized extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go new file mode 100644 index 00000000000..1711765f7ca --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "reflect" + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" +) + +// eval.yaml is the file a user writes, so its keys are the contract. They are +// pinned whole rather than exercised through fixtures: a fixture that stops +// parsing says a test broke, not that a published key was renamed under +// everyone who already wrote one. +// +// The spec's configuration model is the source for every list here. Changing +// one means changing both. + +// yamlKeys reads the yaml tag names off a struct, in declaration order. +func yamlKeys(t *testing.T, v any) []string { + t.Helper() + typ := reflect.TypeOf(v) + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + + var keys []string + for field := range typ.Fields() { + tag := field.Tag.Get("yaml") + if tag == "" || tag == "-" { + continue + } + name := strings.Split(tag, ",")[0] + if name == "" { + continue + } + keys = append(keys, name) + } + return keys +} + +// The top level: catalogs first, then the evals defined over them. +func TestEvalConfigKeys(t *testing.T) { + assert.Equal(t, []string{"datasets", "evaluators", "evals"}, + yamlKeys(t, EvalConfig{}), + "the top-level shape is the spec's configuration model") +} + +// An eval names what it evaluates, what it reads, and how to grade it. +func TestEvalKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{ + "name", "id", "description", "dataset", "source", + "evaluation_level", "max_samples", "evaluators", "target", + }, + yamlKeys(t, Eval{})) +} + +// Every entry in an eval's evaluators: list is a map keyed evaluator:, and the +// spec gives that map exactly five keys. +func TestEvaluatorRefKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{"evaluator", "name", "version", "initialization_parameters", "data_mapping"}, + yamlKeys(t, evalcore.EvaluatorRef{}), + "the spec tabulates these five; a sixth is a promise it does not make") +} + +// source: says where rows come from when they are not a dataset. +func TestSourceDeclKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{ + "type", "lookback_hours", "max_traces", "agent_name", "response_ids", "max_turns", + "agent_version", "start_time", "end_time", + }, + yamlKeys(t, SourceDecl{})) +} + +// The catalogs are named, reusable assets: a name and where it comes from. +func TestCatalogKeys(t *testing.T) { + assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, DatasetDecl{})) + assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, EvaluatorDecl{})) +} + +// The spec's casing table: eval.yaml uses the API's snake_case throughout, so +// a camelCase key would be the one place a reader has to remember an exception. +func TestEveryKeyIsSnakeCase(t *testing.T) { + shapes := map[string]any{ + "EvalConfig": EvalConfig{}, + "Eval": Eval{}, + "SourceDecl": SourceDecl{}, + "Target": Target{}, + "DatasetDecl": DatasetDecl{}, + "EvaluatorDecl": EvaluatorDecl{}, + "EvaluatorRef": evalcore.EvaluatorRef{}, + } + + for name, shape := range shapes { + for _, key := range yamlKeys(t, shape) { + assert.Equalf(t, strings.ToLower(key), key, + "%s.%s is not snake_case; eval.yaml uses the API's spelling throughout", name, key) + assert.NotContainsf(t, key, "-", + "%s.%s uses a dash; the API's convention is underscores", name, key) + } + } +} + +// `target:` always means invoke and `source:` always means where rows come +// from. A trace-backed eval has no target, which is what agent_name under +// source: exists to say. +func TestTargetAndSourceAreDistinct(t *testing.T) { + assert.ElementsMatch(t, []string{"type", "name"}, yamlKeys(t, Target{}), + "the spec's target: is a type and a name; a version there would pin the "+ + "agent an eval invokes, which nothing asks for") + + assert.Contains(t, yamlKeys(t, SourceDecl{}), "agent_name", + "a trace run filters by agent rather than invoking one") + assert.NotContains(t, yamlKeys(t, Target{}), "agent_name", + "the target already names what it invokes") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go new file mode 100644 index 00000000000..643f84c6ceb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -0,0 +1,366 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package project models the eval configuration carried by the +// `host: azure.ai.eval` service entry in azure.yaml. +package project + +import ( + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" +) + +// EvalConfig is one evaluation configuration: the catalogs of reusable assets, +// and every eval defined over them. +// +// It is the body of a single `azure.ai.eval` service entry, pulled in with +// $ref. One file rather than one per eval, because the catalogs are shared: +// two evals over the same dataset should name it once. +// +// How it is stored lives in eval_config_store.go. +type EvalConfig struct { + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` +} + +// DatasetDecl is a catalog entry. A local Source is uploaded on deploy; without +// one the name must already resolve to a registered dataset. +type DatasetDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvaluatorDecl is a catalog entry for a custom evaluator. Built-ins are +// referenced straight from an eval and never declared here. +// +// Source names a `.json` file holding a rubric: a list of weighted scoring +// dimensions. +type EvaluatorDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// Eval is one evaluation defined over the catalogs. +// +// Dataset and Source are alternatives: rows come from a catalog dataset, or +// from a source such as production traces. Target is what gets invoked, and is +// a separate axis — an eval can read traces and invoke nothing. +type Eval struct { + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Source *SourceDecl `yaml:"source,omitempty" json:"source,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` +} + +// SourceDecl says where an eval's rows come from when they are not a dataset. +type SourceDecl struct { + Type string `yaml:"type" json:"type"` + LookbackHours int `yaml:"lookback_hours,omitempty" json:"lookback_hours,omitempty"` + MaxTraces int `yaml:"max_traces,omitempty" json:"max_traces,omitempty"` + AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` + ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` + MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` + // AgentVersion pins which deployment's spans are read. Without it the + // service chooses, and a redeployed agent is evaluated against whichever + // version it picked. + AgentVersion string `yaml:"agent_version,omitempty" json:"agent_version,omitempty"` + // StartTime and EndTime bound the window explicitly. LookbackHours stays + // supported, read as a start bound measured back from EndTime, or from now + // when nothing closes the window. + StartTime string `yaml:"start_time,omitempty" json:"start_time,omitempty"` + EndTime string `yaml:"end_time,omitempty" json:"end_time,omitempty"` +} + +// Source types an eval can read rows from. +const ( + SourceTypeTraces = "traces" + SourceTypeResponses = "responses" +) + +// DefaultScaffoldMaxTraces is the cap init writes on a trace-backed eval, so a +// first run is bounded rather than taking the service's own default of 1000. +// Deleting max_traces from the file restores that default. +const DefaultScaffoldMaxTraces = 20 + +// Target names what the run invokes. +type Target struct { + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` +} + +// Target types the extension can invoke. Absent means nothing is invoked and +// the dataset already carries the answers. +const ( + TargetTypeAgent = "agent" + TargetTypeModel = "model" +) + +// Evaluation levels accepted by the service. The service default is turn. +const ( + EvaluationLevelTurn = "turn" + EvaluationLevelConversation = "conversation" +) + +// EvalNames lists the declared evals in declaration order. +func (c *EvalConfig) EvalNames() []string { + names := make([]string, 0, len(c.Evals)) + for _, e := range c.Evals { + names = append(names, e.Name) + } + return names +} + +// Eval returns the named eval. +// +// An empty name is only answered when the file declares exactly one, because +// guessing which eval a command meant is the kind of mistake that is noticed +// only after it has run. +func (c *EvalConfig) Eval(name string) (*Eval, error) { + if name == "" { + switch len(c.Evals) { + case 0: + return nil, messages.NoEvalsDeclared() + case 1: + return &c.Evals[0], nil + default: + return nil, messages.SeveralEvalsDeclared(len(c.Evals), c.EvalNames()) + } + } + + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], nil + } + } + return nil, messages.EvalNotDeclared(name, c.EvalNames()) +} + +// HasEval reports whether the named eval is declared. Unlike Eval it never +// falls back to "the only one", so callers checking for a collision cannot +// match a differently named entry. +func (c *EvalConfig) HasEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + return true + } + } + return false +} + +// RemoveEval drops the named eval, reporting whether it was there. +func (c *EvalConfig) RemoveEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + c.Evals = append(c.Evals[:i], c.Evals[i+1:]...) + return true + } + } + return false +} + +// DatasetDeclaration returns the catalog entry an eval's `dataset:` names. +func (c *EvalConfig) DatasetDeclaration(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} + +// EvaluatorDeclaration returns the catalog entry an evaluator reference names. +func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true + } + } + return nil, false +} + +// CustomEvaluators are the catalog entries this configuration owns — the ones +// carrying a local source, published before the evals that name them. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, decl := range c.Evaluators { + if decl.Source == "" { + continue + } + owned = append(owned, decl) + } + return owned +} + +// LocalDatasets are the catalog entries carrying a file to upload. +func (c *EvalConfig) LocalDatasets() []DatasetDecl { + var owned []DatasetDecl + for _, decl := range c.Datasets { + if decl.Source == "" { + continue + } + owned = append(owned, decl) + } + return owned +} + +// Validate checks the invariants the provider relies on before it calls the +// service, so failures surface as config errors rather than opaque 4xx. +func (c *EvalConfig) Validate() error { + return c.validate(true) +} + +// ValidateForLookup checks what resolving a declaration by name depends on: a +// readable set of catalogs, and a name that is present and not shared. +// +// What an eval says about itself is left to deploying it, and to the run door, +// which applies the same rules to the entry the run is actually about. Checking +// it here stranded commands that had already been told which eval they meant -- +// `run list --eval ` refused to list anything because a different entry +// was malformed, and the way out was to hand-edit a file the error did not +// mention. +// +// The catalogs stay, because they are the file's shared half: a duplicate +// dataset name makes the lookup this method exists to serve ambiguous, and no +// declaration can be read against a catalog that does not parse into one. +func (c *EvalConfig) ValidateForLookup() error { + return c.validate(false) +} + +func (c *EvalConfig) validate(deploying bool) error { + if err := c.validateCatalogs(); err != nil { + return err + } + // A catalog with no eval is what `generate` leaves behind, and it stays + // that way until `init` wires one. Refusing it on the way to a lookup + // stranded `run --eval ` in a project where `generate` ran first, over + // the absence of a declaration the id did not need. `Eval` answers for the + // case that does need one. + if deploying && len(c.Evals) == 0 { + return messages.AtLeastOneEvalRequired() + } + + seen := map[string]bool{} + substance := map[string]string{} + for i, eval := range c.Evals { + if eval.Name == "" { + return messages.EvalNameRequired(i) + } + if seen[eval.Name] { + return messages.DuplicateEvalName(i, eval.Name) + } + seen[eval.Name] = true + + if !deploying { + // Only what resolving a declaration by name depends on, which is a + // name that is present and not shared. Everything an eval says about + // itself is checked on the way to deploying it, and again at the run + // door on the entry the run is actually about. Enforcing it here + // stranded commands that had already been told which eval they + // meant: one malformed entry stopped `run list --eval ` + // listing anything, and the way out was to hand-edit a file the + // error did not mention. + continue + } + + if err := c.validateEval(i, eval); err != nil { + return err + } + + // Two evals that differ only by name are indistinguishable once + // deployed: the environment records an id against each eval's substance + // so a renamed declaration can find what it already deployed, and a + // shared substance makes that lookup ambiguous. + digest, err := FingerprintGroup(eval) + if err != nil { + return err + } + if first, clash := substance[digest]; clash { + return messages.EvalsIdenticalApartFromName(i, eval.Name, first) + } + substance[digest] = eval.Name + } + return nil +} + +func (c *EvalConfig) validateCatalogs() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return messages.DatasetNameRequired(i) + } + if datasets[d.Name] { + return messages.DuplicateDatasetName(i, d.Name) + } + datasets[d.Name] = true + } + + evaluators := map[string]bool{} + for i, e := range c.Evaluators { + if e.Name == "" { + return messages.EvaluatorNameRequired(i) + } + if evaluators[e.Name] { + return messages.DuplicateEvaluatorName(i, e.Name) + } + evaluators[e.Name] = true + + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return messages.BuiltinNeedsNoCatalogEntry(i, e.Name) + } + // The service assigns an evaluator's version on publish, so a declared + // one cannot be honoured alongside a source: the upload lands on + // whatever comes next and the eval binds that, leaving the pin + // describing a version nothing uses. + if e.Source != "" && e.Version != "" { + return messages.EvaluatorVersionWithSource(i, e.Name) + } + } + return nil +} + +func (c *EvalConfig) validateEval(i int, eval Eval) error { + if err := ValidateRunnable(&eval); err != nil { + return messages.InEvalAt(i, eval.Name, err) + } + if eval.Dataset != "" { + if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { + return messages.DatasetNotInDatasetsCatalog(i, eval.Name, eval.Dataset) + } + } + + if len(eval.Evaluators) == 0 { + return messages.AtLeastOneEvaluatorRequired(i, eval.Name) + } + criteria := map[string]bool{} + for j, ref := range eval.Evaluators { + if ref.Evaluator == "" { + return messages.EvaluatorFieldRequired(i, j) + } + // The criterion name is what identifies a result row, so two rows that + // cannot be told apart are refused here rather than in the results. + criterion := ref.CriterionName() + if criteria[criterion] { + return messages.DuplicateCriterion(i, j, criterion) + } + criteria[criterion] = true + + if ref.IsBuiltin() { + continue + } + if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { + return messages.EvaluatorNotInCatalog(i, j, ref.Evaluator) + } + } + + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go new file mode 100644 index 00000000000..cd41c8d87a8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeFile(t *testing.T, dir, name, body string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) +} + +const oneEvalConfig = "datasets:\n - name: d\n source: ./d.jsonl\n" + +// azure.yaml references one configuration by name. With both files present the +// CLI would edit whichever it preferred while azd up deployed whichever the +// $ref named, and nothing would say so. +func TestOpenEvalConfig_RefusesBothNames(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, oneEvalConfig) + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + _, err := OpenEvalConfig(dir) + + require.Error(t, err) + assert.Contains(t, err.Error(), EvalConfigBase) + assert.Contains(t, err.Error(), LegacyEvalConfigBase) +} + +// The same refusal has to apply on the way out, or generate would append a +// catalog entry to one file while the deployment read the other. +func TestSaveEvalConfig_RefusesBothNames(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, oneEvalConfig) + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + err := SaveEvalConfig(dir, &EvalConfig{}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "only one") +} + +// A project that predates the rename keeps working, and must not silently grow +// a second configuration beside the one it already has. +func TestSaveEvalConfig_WritesBackToTheLegacyFile(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "d", Source: "./d.jsonl"}}, + })) + + assert.FileExists(t, filepath.Join(dir, LegacyEvalConfigBase)) + assert.NoFileExists(t, filepath.Join(dir, EvalConfigBase), + "a legacy project must not grow a second configuration") +} + +// A fresh directory gets the current name. +func TestSaveEvalConfig_WritesTheCurrentNameWhenThereIsNoFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{})) + + assert.FileExists(t, filepath.Join(dir, EvalConfigBase)) + assert.NoFileExists(t, filepath.Join(dir, LegacyEvalConfigBase)) +} + +// A target with no name was scored as though nothing were invoked, which is a +// different evaluation from the one that was written down. +func TestValidate_RefusesATargetWithNoName(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, + "datasets:\n - name: d\n source: ./d.jsonl\n"+ + "evals:\n - name: e\n dataset: d\n"+ + " target:\n type: agent\n"+ + " evaluators:\n - evaluator: builtin.relevance\n") + + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + + err = cfg.Validate() + + require.Error(t, err) + assert.Contains(t, err.Error(), "target.name is required") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go new file mode 100644 index 00000000000..f41a20dcd66 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The config is read by other processes while this one writes it, and +// os.WriteFile truncates before it writes. A reader landing in that window got +// zero bytes -- which parses as a VALID EMPTY CONFIG, not an error -- and would +// then write back a file with every eval missing, reporting success. +// +// Replacing by rename means a reader sees the whole old file or the whole new +// one. This drives a writer and a reader concurrently and asserts the reader +// never observes a config that lost its evals. +func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + + full := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{ + {Name: "first", EvaluationLevel: "turn"}, + {Name: "second", EvaluationLevel: "turn"}, + }, + } + require.NoError(t, SaveEvalConfigTo(path, full)) + + // Every field, not the eval count. A document caught mid-write can still + // parse with two evals while having lost the datasets, or a field off the + // second one, and counting entries reports that as a whole file. Read back + // what a correct read returns and hold every later read to it. + baseline, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, baseline.Evals, 2) + require.Len(t, baseline.Datasets, 1) + + var wg sync.WaitGroup + stop := make(chan struct{}) + // Counted apart, because they want opposite responses. A config observed + // with fields missing is the bug this test guards. A read that failed under + // contention may only mean the retry budget was short on a loaded machine. + // One combined counter, or one example off whichever happened first, leaves + // a run where both occurred looking like whichever won the race. + var readErrors, mismatches int + var firstReadError, firstMismatch string + var replacements int64 + + wg.Go(func() { + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + cfg, err := LoadEvalConfig(path) + if err != nil { + // NOT skipped: "the file does not exist" is exactly what a + // remove-then-rename exposes, and OpenEvalConfig turns it into + // "there is no configuration yet" -- the same loss this guards + // against, by another route. + readErrors++ + if firstReadError == "" { + firstReadError = err.Error() + } + continue + } + if !reflect.DeepEqual(cfg, baseline) { + mismatches++ + if firstMismatch == "" { + firstMismatch = fmt.Sprintf( + "%d evals and %d datasets, wanted %d and %d", + len(cfg.Evals), len(cfg.Datasets), + len(baseline.Evals), len(baseline.Datasets)) + } + } + } + close(stop) + }) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + if SaveEvalConfigTo(path, full) == nil { + atomic.AddInt64(&replacements, 1) + } + } + } + }) + wg.Wait() + + require.NotZero(t, atomic.LoadInt64(&replacements), + "the writer has to have replaced the file, or nothing was under test") + assert.Zerof(t, readErrors+mismatches, + "over %d replacements a concurrent reader saw %d incomplete configs (first: %s) "+ + "and %d failed reads (first: %s)", + atomic.LoadInt64(&replacements), + mismatches, orNone(firstMismatch), + readErrors, orNone(firstReadError)) +} + +// orNone keeps an absent example from reading as an empty one. +func orNone(s string) string { + if s == "" { + return "none" + } + return s +} + +// OpenEvalConfig maps a missing file to "no configuration yet", which callers +// answer by writing a fresh one. So a replacement that momentarily unlinks the +// destination is as destructive as one that truncates it, and this pins the +// window closed from that side too. +func TestOpenEvalConfigNeverSeesTheFileVanish(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + full := &EvalConfig{Evals: []Eval{{Name: "first", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, full)) + + var wg sync.WaitGroup + stop := make(chan struct{}) + var vanished, replacements int64 + + wg.Go(func() { + // Wall clock, not an iteration count. Three hundred os.Stat calls take + // microseconds, which is not long enough for the writer to be scheduled + // even once -- the test passed against the unlinking version it was + // written to catch. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + atomic.AddInt64(&vanished, 1) + } + } + close(stop) + }) + wg.Go(func() { + for { + select { + case <-stop: + return + default: + if SaveEvalConfigTo(path, full) == nil { + atomic.AddInt64(&replacements, 1) + } + } + } + }) + wg.Wait() + + require.NotZero(t, atomic.LoadInt64(&replacements), + "the writer has to have replaced the file, or nothing was under test") + assert.Zerof(t, vanished, "the config was absent %d times during replacement", vanished) +} + +// The replacement must leave the file complete and parseable. +func TestSaveEvalConfigRoundTripsThroughTheRename(t *testing.T) { + path := filepath.Join(t.TempDir(), "azure.eval.yaml") + want := &EvalConfig{Evals: []Eval{{Name: "only", EvaluationLevel: "turn"}}} + + // A different first write, so the second one has something to replace. + // Writing the same payload twice passes even if the second save silently + // left the original file where it was, which is the case worth catching. + first := &EvalConfig{Evals: []Eval{{Name: "replaced", EvaluationLevel: "conversation"}}} + require.NoError(t, SaveEvalConfigTo(path, first)) + require.NoError(t, SaveEvalConfigTo(path, want)) + + got, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, got.Evals, 1) + assert.Equal(t, "only", got.Evals[0].Name, "the second save has to have replaced the first") + assert.Equal(t, "turn", got.Evals[0].EvaluationLevel) + + // The temporary file is this function's business and must not be left over. + entries, err := os.ReadDir(filepath.Dir(path)) + require.NoError(t, err) + assert.Len(t, entries, 1, "the rename must not leave a temporary file behind") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go new file mode 100644 index 00000000000..6f91532502f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strings" + + "azureaieval/internal/pkg/evalcore" +) + +// goTypeInField matches what yaml.KnownFields reports for an unrecognized key: +// `line 7: field evaulators not found in type project.Eval`. The Go type is an +// implementation detail, and the reader is editing a YAML file by hand, which +// is the documented way to use one. +var goTypeInField = regexp.MustCompile(`field (\S+) not found in type (\S+)`) + +// explainUnknownKeys rewrites a decode failure into the file's own vocabulary, +// naming the near-miss when there is one. +func explainUnknownKeys(err error) error { + text := err.Error() + if !goTypeInField.MatchString(text) { + return err + } + + // Nothing at the top level was recognized, so this is another tool's file + // rather than a typo in one of ours. `azd ai agent eval` writes an eval.yaml + // of its own, and suggesting a near-miss for each of its keys in turn would + // walk the reader into rewriting it a line at a time. + if topLevelKeysAllUnknown(text) { + return errUnrecognizedEvalConfig + } + + lines := make([]string, 0, 4) + for line := range strings.SplitSeq(text, "\n") { + m := goTypeInField.FindStringSubmatch(line) + if m == nil { + continue + } + key, goType := m[1], m[2] + rewritten := fmt.Sprintf("unknown key %q", key) + if near := nearestKey(key, keysOfType(goType)); near != "" { + rewritten += fmt.Sprintf(`; did you mean %q?`, near) + } + // The yaml prefix carries the line number, which is the useful half. + if prefix, _, ok := strings.Cut(strings.TrimSpace(line), ": field "); ok { + rewritten = prefix + ": " + rewritten + } + lines = append(lines, rewritten) + } + if len(lines) == 0 { + return err + } + return fmt.Errorf("%s", strings.Join(lines, "\n")) +} + +var errUnrecognizedEvalConfig = errors.New( + "none of this file's top-level keys are ones an eval configuration declares, " + + "so this is not one. `azd ai agent eval` writes an eval.yaml of its own with " + + "a different shape, and runs it with `azd ai agent eval run`") + +// topLevelKeysAllUnknown reports a file whose top-level shape is not this one. +// +// Only top-level rejections count. A nested one does not disqualify the check: +// another tool's file can still have a key named like one of ours, and the +// mismatch inside it then reports against the nested type rather than the +// config. The threshold is the number of keys a configuration declares, so one +// stray key beside recognized ones stays a typo. +func topLevelKeysAllUnknown(text string) bool { + known := keysOfType("project.EvalConfig") + rejected := 0 + for line := range strings.SplitSeq(text, "\n") { + m := goTypeInField.FindStringSubmatch(line) + if m == nil || m[2] != "project.EvalConfig" { + continue + } + rejected++ + } + return rejected > 0 && rejected >= len(known) +} + +// keysOfType lists the YAML keys a declaration accepts. +func keysOfType(goType string) []string { + var v any + switch goType { + case "project.EvalConfig": + v = EvalConfig{} + case "project.Eval": + v = Eval{} + case "project.DatasetDecl": + v = DatasetDecl{} + case "project.EvaluatorDecl": + v = EvaluatorDecl{} + case "project.Target": + v = Target{} + case "project.SourceDecl": + v = SourceDecl{} + case "evalcore.EvaluatorRef": + v = evalcore.EvaluatorRef{} + default: + return nil + } + + t := reflect.TypeOf(v) + keys := make([]string, 0, t.NumField()) + for field := range t.Fields() { + tag, _, _ := strings.Cut(field.Tag.Get("yaml"), ",") + if tag != "" && tag != "-" { + keys = append(keys, tag) + } + } + return keys +} + +// nearestKey returns the closest known key, or "" when nothing is close enough +// to be worth suggesting. A third of the length is the budget, so `evaulators` +// finds `evaluators` while `banana` suggests nothing. +func nearestKey(key string, known []string) string { + best, bestDist := "", len(key)/3+1 + for _, k := range known { + if d := editDistance(key, k); d < bestDist { + best, bestDist = k, d + } + } + return best +} + +// editDistance is Levenshtein over two short identifiers. +func editDistance(a, b string) int { + prev := make([]int, len(b)+1) + curr := make([]int, len(b)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(a); i++ { + curr[0] = i + for j := 1; j <= len(b); j++ { + cost := 1 + if a[i-1] == b[j-1] { + cost = 0 + } + curr[j] = min(min(curr[j-1]+1, prev[j]+1), prev[j-1]+cost) + } + prev, curr = curr, prev + } + return prev[len(b)] +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go new file mode 100644 index 00000000000..fc3338e994e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Hand-editing the file is the documented way to use one, so a mistyped key has +// to read as a mistyped key rather than as a Go type the reader has never seen. +func TestExplainUnknownKeys(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 7: field evaulators not found in type project.Eval")).Error() + + assert.Contains(t, got, `unknown key "evaulators"`) + assert.Contains(t, got, `did you mean "evaluators"?`) + assert.Contains(t, got, "line 7", "the line number is the useful half of what yaml said") + assert.NotContains(t, got, "project.Eval", "the Go type is an implementation detail") +} + +// Nothing close enough is worth suggesting; a wrong suggestion is worse than +// none. +func TestExplainUnknownKeys_NoNearMiss(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 3: field banana not found in type project.Eval")).Error() + + assert.Contains(t, got, `unknown key "banana"`) + assert.NotContains(t, got, "did you mean") +} + +// Anything that is not an unknown-key failure is passed through untouched. +func TestExplainUnknownKeys_LeavesOtherErrors(t *testing.T) { + original := errors.New("yaml: line 2: did not find expected key") + assert.Equal(t, original, explainUnknownKeys(original)) +} + +func TestKeysOfTypeCoversTheDeclarations(t *testing.T) { + assert.Contains(t, keysOfType("project.Eval"), "evaluators") + assert.Contains(t, keysOfType("project.EvalConfig"), "datasets") + assert.Contains(t, keysOfType("project.DatasetDecl"), "source") + assert.Empty(t, keysOfType("project.Unknown")) +} + +// `azd ai agent eval` writes an eval.yaml of its own with an entirely different +// shape. Suggesting a near-miss for each of its keys in turn would walk the +// reader into rewriting another tool's file a line at a time. +// +// Its `evaluators` key happens to share a name with ours, so the mismatch +// inside it reports against the nested type. That must not disqualify the check. +func TestExplainUnknownKeys_AnotherToolsFile(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n" + + " line 1: field name not found in type project.EvalConfig\n" + + " line 2: field agent not found in type project.EvalConfig\n" + + " line 6: field dataset not found in type project.EvalConfig\n" + + " line 13: field local_uri not found in type project.EvaluatorDecl\n" + + " line 14: field options not found in type project.EvalConfig\n" + + " line 16: field max_samples not found in type project.EvalConfig")).Error() + + assert.Contains(t, got, "not one") + assert.Contains(t, got, "azd ai agent eval run", "the reader is told where the file does belong") + assert.NotContains(t, got, "did you mean", + "suggesting a fix per key sends the reader down the wrong path entirely") +} + +// One stray key beside recognized ones is still a typo, so the suggestion stands. +func TestExplainUnknownKeys_OneStrayTopLevelKey(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 1: field datsets not found in type project.EvalConfig")).Error() + + assert.Contains(t, got, `did you mean "datasets"?`) + assert.NotContains(t, got, "does not look like") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go new file mode 100644 index 00000000000..136f3addeb5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "azureaieval/internal/messages" + + "github.com/gofrs/flock" +) + +// configLockTimeout bounds the wait for another process's read-modify-write. +// Nothing that holds this lock waits on a person -- the evaluator prompt is +// deliberately outside it -- so a wait longer than this is a stale lock rather +// than contention. +const configLockTimeout = 30 * time.Second + +// evalConfigLockName is the lock file, beside the configuration it guards. +// +// Not in the OS temp directory, which looked tidier and was wrong twice over: a +// lock file there is created 0600 by whoever runs first, so a second user on +// the same machine can never open it and silently never locks; and two +// containers bind-mounting one project have separate temp directories, so they +// never see each other's lock at all. Beside the config it shares the project's +// lifetime, permissions and mount, and the `git status` noise that argued for +// temp is answered by ignoreLockFile. +const evalConfigLockName = ".azure.eval.lock" + +// LockEvalConfig serializes read-modify-write on the configuration across +// processes, returning the release function. +// +// Updating the configuration means reading the file, adding an entry and +// writing it back. Two processes doing that at once can both read the same +// state, and the second write then drops the first one's entry -- a lost update +// that reports success on both sides. The atomic write stops a reader seeing a +// half-written file; it cannot stop this. +// +// Advisory and best-effort: a lock that could not be taken is reported and the +// work goes ahead, because failing a scaffold over a lock file would be worse +// than the lost update it guards against. Reported on stderr rather than +// through log, which is pointed at io.Discard unless --debug -- an earlier +// version logged it and was therefore exactly as silent as saying nothing. +func LockEvalConfig(ctx context.Context, evalDir string) (func(), error) { + if ctx == nil { + // cobra hands a nil context to a command that was not run through + // Execute, and waiting on nil panics. + ctx = context.Background() + } + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return nil, messages.Creating(evalDir, err) + } + + lock := flock.New(filepath.Join(evalDir, evalConfigLockName)) + waitCtx, cancel := context.WithTimeout(ctx, configLockTimeout) + defer cancel() + + locked, err := lock.TryLockContext(waitCtx, 50*time.Millisecond) + if err != nil || !locked { + // Being cancelled is not the same as the lock being busy. The advisory + // behavior below exists so a held lock cannot fail a scaffold; carrying + // it into Ctrl-C would go on to rewrite the configuration the user just + // asked to stop. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + fmt.Fprint(os.Stderr, messages.Warning(messages.ConfigLockUnavailable(evalDir, err))) + return func() {}, nil + } + // Only once the file is ours: a lock that was never taken has no artifact + // to hide, and writing into a directory the user commits is not something + // to do on the way past. + ignoreLockFile(evalDir) + return func() { _ = lock.Unlock() }, nil +} + +// ignoreLockFile keeps the lock out of `git status`, which is the one thing the +// OS temp directory had going for it. +// +// Only when there is no .gitignore of its own to respect: editing a file the +// user maintains is not this function's business, and a visible lock file is a +// far smaller problem than a surprising edit. +func ignoreLockFile(evalDir string) { + path := filepath.Join(evalDir, ".gitignore") + if _, err := os.Stat(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return + } + _ = os.WriteFile(path, []byte(evalConfigLockName+"\n"), 0o600) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go new file mode 100644 index 00000000000..426b45f24c5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The lock is advisory: a scaffold must not fail because a lock file could +// not be taken, and an earlier version reported that case only through log, +// which is pointed at io.Discard unless --debug -- exactly as silent as saying +// nothing. It now reports on stderr. +func TestLockEvalConfigIsTakenAndReleased(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + require.NotNil(t, unlock) + unlock() + + // And it is reusable once released. + unlock2, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + require.NotNil(t, unlock2) + unlock2() +} + +// The lock lives beside the configuration it guards, not in the OS temp +// directory. Temp looked tidier and was wrong twice over: the file is created +// 0600 by whoever runs first, so a second user on the same machine can never +// open it and silently never locks; and two containers bind-mounting one +// project have separate temp directories, so they never see each other's lock. +func TestLockEvalConfigLivesBesideTheConfig(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + _, err = os.Stat(filepath.Join(dir, evalConfigLockName)) + assert.NoError(t, err, "the lock belongs in the directory it guards") +} + +// Beside the config means inside a directory the user commits, so the lock has +// to keep itself out of `git status` -- which is the one thing the temp +// directory had going for it. +func TestLockEvalConfigIgnoresItself(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + body, err := os.ReadFile(filepath.Join(dir, ".gitignore")) + require.NoError(t, err) + assert.Contains(t, string(body), evalConfigLockName) +} + +// A .gitignore the user maintains is theirs. Appending to it is a surprising +// edit, and a visible lock file is the far smaller problem. +func TestLockEvalConfigLeavesAnExistingGitignoreAlone(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, os.MkdirAll(dir, 0o750)) + theirs := filepath.Join(dir, ".gitignore") + require.NoError(t, os.WriteFile(theirs, []byte("*.local\n"), 0o600)) + + unlock, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + body, err := os.ReadFile(theirs) + require.NoError(t, err) + assert.Equal(t, "*.local\n", string(body), "the user's file is not ours to edit") +} + +// cobra hands a nil context to a command that was not run through Execute, and +// waiting on a nil context panics. +func TestLockEvalConfigToleratesANilContext(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + //nolint:staticcheck // the nil context is the case under test + unlock, err := LockEvalConfig(nil, dir) + require.NoError(t, err) + require.NotNil(t, unlock) + unlock() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go new file mode 100644 index 00000000000..b74ab1cdf96 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const minimalConfig = "datasets:\n - name: golden\n source: ./datasets/golden.jsonl\n" + +// The file is named for azd, the way azure.yaml is. +func TestEvalConfigPath_IsTheAzdPrefixedName(t *testing.T) { + assert.Equal(t, filepath.Join("evals", "azure.eval.yaml"), EvalConfigPath("evals")) +} + +// A project written before the rename has a checked-in eval.yaml and an +// azure.yaml $ref pointing at it. Reading has to find it, or every such project +// silently looks empty. +func TestOpenEvalConfig_ReadsALegacyFile(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + + cfg, err := OpenEvalConfig(dir) + + require.NoError(t, err) + require.NotNil(t, cfg, "a legacy configuration is still a configuration") + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "golden", cfg.Datasets[0].Name) +} + +// Writing back into such a project has to update the file it already +// references. Creating azure.eval.yaml beside it would leave the one azure.yaml +// $refs untouched, so the entry would be invisible to azd up. +func TestSaveEvalConfig_WritesBackOverALegacyFile(t *testing.T) { + dir := t.TempDir() + legacy := filepath.Join(dir, LegacyEvalConfigBase) + require.NoError(t, os.WriteFile(legacy, []byte(minimalConfig), 0o600)) + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "added", Source: "./datasets/added.jsonl"}}, + })) + + body, err := os.ReadFile(legacy) + require.NoError(t, err) + assert.Contains(t, string(body), "added") + + _, err = os.Stat(EvalConfigPath(dir)) + assert.True(t, os.IsNotExist(err), + "a second configuration beside the one azure.yaml references would be inert") +} + +// A directory holding both names is refused, not silently resolved. Preferring +// one is the dangerous answer: azure.yaml $refs a single file by name, so the +// CLI would edit one configuration while `azd up` deployed the other. +// +// This used to assert the preference instead, and `eval create` -- the one +// command that asks for the path rather than the parsed configuration -- read +// one of the two without a word while `run`, `init` and `generate` all refused. +func TestResolveEvalConfigPath_RefusesADirectoryHoldingBoth(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte(minimalConfig), 0o600)) + + path, err := ResolveEvalConfigPath(dir) + require.Error(t, err, "both names present has no single right answer") + assert.Empty(t, path) + assert.Contains(t, err.Error(), LegacyEvalConfigBase) + assert.Contains(t, err.Error(), EvalConfigBase) +} + +// The naming preference itself still holds for the callers that have already +// applied the guard: with only the legacy file there, that is the file a save +// writes back over. +func TestResolvedConfigPath_PrefersTheCurrentName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte(minimalConfig), 0o600)) + + assert.Equal(t, EvalConfigPath(dir), resolvedConfigPath(dir)) +} + +// An empty directory resolves to the current name, which is what a first +// generate creates. +func TestResolveEvalConfigPath_EmptyDirectoryUsesTheCurrentName(t *testing.T) { + dir := t.TempDir() + + path, err := ResolveEvalConfigPath(dir) + require.NoError(t, err) + assert.Equal(t, EvalConfigPath(dir), path) +} + +// Only the legacy file present resolves to it, so a project that has not +// migrated keeps working. +func TestResolveEvalConfigPath_LegacyOnlyResolvesToLegacy(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + + path, err := ResolveEvalConfigPath(dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, LegacyEvalConfigBase), path) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go new file mode 100644 index 00000000000..8b60d4073c8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A config can be read-only: a Perforce or TFVC checkout marks files that way +// by default, as does `attrib +R` and some archive extractions. +// +// This used to work by accident. The replacement removed the destination first, +// and os.Remove clears FILE_ATTRIBUTE_READONLY and retries the delete, so the +// attribute never reached the rename. Dropping the unlink -- which was right, +// because it opened a window where the config did not exist -- took that repair +// with it, and Windows reports a rename onto a read-only destination with the +// same errno as one a reader holds open, so it cannot be told apart earlier. +func TestSaveEvalConfigReplacesAReadOnlyFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "azure.eval.yaml") + first := &EvalConfig{Evals: []Eval{{Name: "first", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, first)) + require.NoError(t, os.Chmod(path, 0o400)) + + second := &EvalConfig{Evals: []Eval{{Name: "second", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, second), + "a read-only config has to be replaceable, as it was before the rename") + + got, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, got.Evals, 1) + assert.Equal(t, "second", got.Evals[0].Name, "and the new content has to be there") +} + +// The retry exists for a window measured in microseconds. A file that is +// genuinely unreadable shares an errno with that window on Windows, so it pays +// the budget before it is reported -- which is only acceptable while the budget +// stays small. +func TestContentionBudgetsStaySmall(t *testing.T) { + assert.LessOrEqual(t, readRetryBudget.Milliseconds(), int64(250), + "every unreadable file pays this, and ReadFileNoBOM reads one per evaluator") + assert.LessOrEqual(t, renameRetryBudget.Milliseconds(), int64(500), + "a read-only destination waits this out before the attribute is cleared") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go new file mode 100644 index 00000000000..27cbcc1fe27 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `init` and `generate` both read the configuration, add an entry and write the +// whole file back. Anything the round trip cannot carry is deleted from a file +// the developer wrote and is expected to keep editing. +func TestEvalConfigRoundTripKeepsWhatTheAuthorWrote(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + authored := `evals: + - name: support-agent-eval + target: + type: agent + name: support-agent + dataset: golden + evaluators: + - evaluator: builtin.task_adherence +datasets: + - name: golden + source: ./datasets/golden.jsonl +` + require.NoError(t, os.WriteFile(path, []byte(authored), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, SaveEvalConfigTo(path, cfg)) + + reloaded, err := LoadEvalConfig(path) + require.NoError(t, err) + + require.Len(t, reloaded.Evals, 1) + assert.Equal(t, "support-agent-eval", reloaded.Evals[0].Name) + require.NotNil(t, reloaded.Evals[0].Target, "the target survived the rewrite") + assert.Equal(t, "agent", reloaded.Evals[0].Target.Type) + assert.Equal(t, "support-agent", reloaded.Evals[0].Target.Name) + require.Len(t, reloaded.Datasets, 1) + assert.Equal(t, "golden", reloaded.Datasets[0].Name) +} + +// A hand-edited configuration is the normal way to use this file, and a +// misspelled key used to be read as nothing at all: `agent: support-agent` +// under `target:` left an empty target, and the run failed later with a message +// about the target rather than about the typo that caused it. +func TestEvalConfigRefusesAKeyItDoesNotKnow(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + require.NoError(t, os.WriteFile(path, []byte(`evals: + - name: support-agent-eval + target: + agent: support-agent +`), 0o600)) + + _, err := LoadEvalConfig(path) + + require.Error(t, err, "a key the extension does not know is a typo, not a no-op") + assert.Contains(t, err.Error(), "agent") +} + +// The keys the extension does know must still load, or strictness would break +// every configuration it writes itself. +func TestEvalConfigAcceptsEveryKeyItWrites(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + require.NoError(t, os.WriteFile(path, []byte(`datasets: + - name: golden + source: ./datasets/golden.jsonl + version: "2" +evaluators: + - name: quality + source: ./evaluators/quality.json +evals: + - name: e + id: eval_1 + description: grades support answers + dataset: golden + evaluation_level: turn + max_samples: 15 + evaluators: + - evaluator: builtin.task_adherence + target: + type: agent + name: support-agent +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, 15, cfg.Evals[0].MaxSamples) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go new file mode 100644 index 00000000000..aefd74e551d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -0,0 +1,305 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "bytes" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "syscall" + "time" + + "azureaieval/internal/messages" + + "go.yaml.in/yaml/v3" +) + +// This file is the only place that knows how the configuration is stored: the +// directory it lives in, what the file is called, and how it is parsed and +// serialized. Everything else works with *EvalConfig, so changing the on-disk +// shape stays a local edit. + +// DefaultEvalDir is where init writes the configuration and its artifacts. +const DefaultEvalDir = "evals" + +// EvalConfigBase is the single configuration file inside that directory. +// +// Prefixed for azd, the way azure.yaml is: eval.yaml is generic enough to +// collide with an unrelated tool's file in the same folder, and the prefix says +// whose it is. +const EvalConfigBase = "azure.eval.yaml" + +// LegacyEvalConfigBase is what the file was called before it was named for azd. +// Read, never written: a project that already has one keeps working, and does +// not silently grow a second configuration beside it. +const LegacyEvalConfigBase = "eval.yaml" + +// EvalConfigPath is the configuration file inside an eval directory. It is +// exported for error messages and for the azure.yaml $ref; readers should +// prefer OpenEvalConfig. +func EvalConfigPath(evalDir string) string { + return filepath.Join(evalDir, EvalConfigBase) +} + +// ResolveEvalConfigPath is the configuration this directory actually holds: +// the current name, or the legacy one when that is the only file there. +// +// It refuses a directory holding both, rather than leaving that to the caller. +// The rule used to live in OpenEvalConfig alone, so `eval create` -- which +// needs the path rather than the parsed configuration -- resolved one silently +// while `run`, `init` and `generate` all refused. Returning an error is what +// makes the guard unavoidable: there is no longer a way to ask this question +// and not be told. +func ResolveEvalConfigPath(evalDir string) (string, error) { + if err := checkOneConfig(evalDir); err != nil { + return "", err + } + return resolvedConfigPath(evalDir), nil +} + +// resolvedConfigPath is the naming rule on its own, for the two functions that +// have already applied the guard. +func resolvedConfigPath(evalDir string) string { + current := EvalConfigPath(evalDir) + if _, err := os.Stat(current); err == nil { + return current + } + legacy := filepath.Join(evalDir, LegacyEvalConfigBase) + if _, err := os.Stat(legacy); err == nil { + return legacy + } + return current +} + +// checkOneConfig refuses a directory holding both names. +// +// Preferring one silently is the dangerous answer: `azure.yaml` `$ref`s a +// single file by name, so the CLI would edit one configuration while `azd up` +// deployed the other, and nothing would say so. +func checkOneConfig(evalDir string) error { + current := EvalConfigPath(evalDir) + legacy := filepath.Join(evalDir, LegacyEvalConfigBase) + if _, err := os.Stat(current); err != nil { + return nil + } + if _, err := os.Stat(legacy); err != nil { + return nil + } + return messages.AmbiguousEvalConfig(current, legacy) +} + +// OpenEvalConfig reads the configuration under evalDir. +// +// A missing file returns (nil, nil): generate runs before init, so "no +// configuration yet" is an ordinary state rather than a failure. +func OpenEvalConfig(evalDir string) (*EvalConfig, error) { + if err := checkOneConfig(evalDir); err != nil { + return nil, err + } + cfg, err := LoadEvalConfig(resolvedConfigPath(evalDir)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return cfg, err +} + +// LoadEvalConfig reads a configuration from an explicit path. The path is used +// verbatim, relative to the process working directory — never re-rooted. +// +// Decoded strictly: a key this extension does not know is a typo, and reading +// it as nothing leaves a configuration that looks fine and fails later +// somewhere else. `agent:` written under `target:` instead of `type:`/`name:` +// used to produce an empty target and a run that complained about the target. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := ReadFileNoBOM(path) + if err != nil { + return nil, messages.ReadingEvalConfig(path, err) + } + return DecodeEvalConfig(data, path) +} + +// DecodeEvalConfig is the one strict decoder, so every route into a +// configuration reports a mistyped key the same way. +// +// `azd up` reads the configuration through the service entry rather than off +// disk, and that route used json.Unmarshal, which drops unknown keys in +// silence. The same typo was therefore named by `azd ai eval run` and ignored +// by `azd up`. The name is what the diagnostic points at. +func DecodeEvalConfig(data []byte, name string) (*EvalConfig, error) { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + + var cfg EvalConfig + if err := decoder.Decode(&cfg); err != nil { + // An empty file is a configuration with nothing in it, not a parse + // failure: `generate` writes one before it has anything to record. + if errors.Is(err, io.EOF) { + return &cfg, nil + } + return nil, messages.ParsingEvalConfig(name, explainUnknownKeys(err)) + } + return &cfg, nil +} + +// SaveEvalConfig writes cfg as the configuration under evalDir, creating the +// directory when it does not exist yet. +// +// Writes back over a legacy eval.yaml when that is the file the project has, so +// a generate into an existing project updates the configuration it already +// references rather than leaving an inert second one beside it. +func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := checkOneConfig(evalDir); err != nil { + return err + } + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return messages.Creating(evalDir, err) + } + return SaveEvalConfigTo(resolvedConfigPath(evalDir), cfg) +} + +// SaveEvalConfigTo writes cfg over an explicit path, for callers that already +// resolved one. +// +// The replacement is atomic because os.WriteFile truncates first, and this file +// is read by other processes. A reader landing inside that window sees zero +// bytes, and a zero-byte config parses as a valid empty one rather than as an +// error, so it would go on to write back a configuration with every eval +// missing. Renaming into place means a reader sees either the whole old file or +// the whole new one. +func SaveEvalConfigTo(path string, cfg *EvalConfig) error { + body, err := yaml.Marshal(cfg) + if err != nil { + return messages.SerializingEvalConfig(err) + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".azd-eval-config-*") + if err != nil { + return messages.WritingEvalConfig(path, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Close(); err != nil { + return messages.WritingEvalConfig(path, err) + } + // Straight over the destination, and never by unlinking it first. Windows + // refuses a rename while a reader holds the destination open, so the + // obvious fallback -- remove, then rename -- turns a collision into a + // window where the config does not exist, and OpenEvalConfig reports a + // missing file as "no configuration yet", which callers answer by writing a + // fresh one. That is the same data loss this function exists to prevent. + // Contention is measured in microseconds, so it is waited out instead. + if err := ReplaceFile(tmpName, path); err != nil { + return messages.WritingEvalConfig(path, err) + } + return nil +} + +// ReplaceFile moves a freshly written temporary file over a destination. +// +// Never by unlinking the destination first, which is the obvious shape and is +// wrong twice over. Windows refuses a rename while a reader holds the +// destination open, so remove-then-rename turns a collision into a window where +// the file does not exist -- and a config that momentarily does not exist reads +// as "no configuration yet", which callers answer by writing a fresh empty one. +// Contention is measured in microseconds, so it is waited out instead. +// +// The unlink was doing one thing worth keeping: os.Remove clears a read-only +// attribute and retries, so a file marked read-only (a Perforce or TFVC +// checkout, `attrib +R`, some archive extractions) could still be replaced. +// Windows reports a rename onto a read-only destination with the same errno as +// one a reader holds open, so the two cannot be told apart before the wait. +func ReplaceFile(from, to string) error { + err := renameOverContention(from, to) + if err == nil { + return nil + } + if !clearReadOnly(to) { + return err + } + return os.Rename(from, to) +} + +// clearReadOnly drops a read-only attribute, reporting whether it had one to +// drop. os.Chmod is what carries FILE_ATTRIBUTE_READONLY on Windows. +func clearReadOnly(path string) bool { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm()&0o200 != 0 { + return false + } + return os.Chmod(path, info.Mode().Perm()|0o200) == nil +} + +// The budgets are deliberately different. A replacement window is measured in +// microseconds, so neither needs to be generous -- and every millisecond here +// is also charged to a file that is genuinely unreadable, because Windows +// reports "someone has this open" and "you may not have this" as one errno. +const ( + renameRetryBudget = 500 * time.Millisecond + readRetryBudget = 250 * time.Millisecond +) + +func renameOverContention(from, to string) error { + deadline := time.Now().Add(renameRetryBudget) + delay := time.Millisecond + for { + err := os.Rename(from, to) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} + +// isSharingContention reports the errors Windows raises while another handle is +// open. It cannot be precise: renaming onto a destination a reader holds open +// and renaming onto one the caller may not touch both report ERROR_ACCESS_DENIED, +// so a genuine permission failure is waited on before it is reported. The +// budget is what keeps that wait short enough to be worth the trade. +func isSharingContention(err error) bool { + if err == nil || errors.Is(err, os.ErrNotExist) { + return false + } + if runtime.GOOS != "windows" { + return false + } + var errno syscall.Errno + if !errors.As(err, &errno) { + return false + } + // ERROR_ACCESS_DENIED and ERROR_SHARING_VIOLATION. + return errno == 5 || errno == 32 +} + +// readFileOverContention reads a file that another process may be replacing. +func readFileOverContention(path string) ([]byte, error) { + deadline := time.Now().Add(readRetryBudget) + delay := time.Millisecond + for { + body, err := os.ReadFile(path) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return body, err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go new file mode 100644 index 00000000000..ba81f5fcdf3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -0,0 +1,481 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// sampleEvalConfig is the shape the spec documents for evals/eval.yaml: two +// catalogs, then the evals defined over them. +const sampleEvalConfig = ` +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + - name: prod-registered + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evals: + - name: support-agent-smoke + description: Quality gate for the support agent + dataset: support-golden + evaluation_level: conversation + max_samples: 100 + evaluators: + - evaluator: builtin.task_adherence + - evaluator: support-quality + name: quality_strict + initialization_parameters: + deployment_name: gpt-4.1-nano + target: + type: agent + name: support-agent + + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence +` + +func loadFromString(t *testing.T, body string) *EvalConfig { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(EvalConfigPath(dir), []byte(body), 0o600)) + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + return cfg +} + +func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.Len(t, cfg.Datasets, 2) + require.Equal(t, "support-golden", cfg.Datasets[0].Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "1", cfg.Datasets[0].Version) + + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "support-quality", cfg.Evaluators[0].Name) + + require.Equal(t, []string{"support-agent-smoke", "support-agent-trace-eval"}, cfg.EvalNames()) +} + +// One file holds many evals, and each is selected by its own name. +func TestEval_SelectsByName(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "support-golden", eval.Dataset) + require.Equal(t, "Quality gate for the support agent", eval.Description) + require.Equal(t, EvaluationLevelConversation, eval.EvaluationLevel) + require.Equal(t, 100, eval.MaxSamples) + require.Len(t, eval.Evaluators, 2) + require.Equal(t, TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) +} + +// A trace-backed eval invokes nothing, so agent_name filters rather than targets. +func TestEval_TraceSourceHasNoTarget(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-trace-eval") + require.NoError(t, err) + require.Nil(t, eval.Target) + require.Equal(t, SourceTypeTraces, eval.Source.Type) + require.Equal(t, "support-agent", eval.Source.AgentName) + require.Equal(t, 20, eval.Source.MaxTraces) +} + +// An unnamed selection is only answered when the file declares exactly one, +// because guessing which eval a command meant is noticed only after it runs. +func TestEval_UnnamedIsAmbiguousWithSeveral(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("") + require.ErrorContains(t, err, "--eval") + require.ErrorContains(t, err, "support-agent-trace-eval") + + single := loadFromString(t, "evals:\n - name: only\n evaluators:\n - evaluator: builtin.relevance\n") + eval, err := single.Eval("") + require.NoError(t, err) + require.Equal(t, "only", eval.Name) +} + +func TestEval_UnknownNameNamesWhatIsDeclared(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("nope") + require.ErrorContains(t, err, "is not declared") + require.ErrorContains(t, err, "support-agent-smoke") +} + +// HasEval never falls back to "the only one", so a collision check cannot match +// a differently named entry. +func TestHasEvalAndRemoveEval(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.True(t, cfg.HasEval("support-agent-smoke")) + require.False(t, cfg.HasEval("nope")) + require.False(t, cfg.HasEval("")) + + require.True(t, cfg.RemoveEval("support-agent-smoke")) + require.False(t, cfg.HasEval("support-agent-smoke")) + require.Equal(t, []string{"support-agent-trace-eval"}, cfg.EvalNames()) + require.False(t, cfg.RemoveEval("support-agent-smoke")) +} + +// Only catalog entries carrying a local source are this config's to publish. +// One without a source already exists on the project. +func TestCustomEvaluatorsAndLocalDatasets_OnlyOwnLocalSources(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + owned := cfg.CustomEvaluators() + require.Len(t, owned, 1) + require.Equal(t, "support-quality", owned[0].Name) + require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) + + local := cfg.LocalDatasets() + require.Len(t, local, 1) + require.Equal(t, "support-golden", local[0].Name, + "prod-registered has no source, so it is already on the project") +} + +func TestDeclarationLookups(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + ds, ok := cfg.DatasetDeclaration("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + + _, ok = cfg.DatasetDeclaration("missing") + require.False(t, ok) + + ev, ok := cfg.EvaluatorDeclaration("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators/support-quality.json", ev.Source) +} + +// The configuration must survive a write/read cycle, because init and generate +// both append to a file they just read. +func TestEvalConfig_RoundTripsThroughTheStore(t *testing.T) { + dir := t.TempDir() + cfg := loadFromString(t, sampleEvalConfig) + + require.NoError(t, SaveEvalConfig(dir, cfg)) + back, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.Equal(t, cfg, back) +} + +// A missing file is an ordinary state: generate runs before init. +func TestOpenEvalConfig_MissingIsNotAnError(t *testing.T) { + cfg, err := OpenEvalConfig(t.TempDir()) + require.NoError(t, err) + require.Nil(t, cfg) +} + +// SaveEvalConfig creates the directory, so generate can record an artifact in a +// project that has never run init. +func TestSaveEvalConfig_CreatesTheDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "generated", Source: "./datasets/generated.jsonl"}}, + })) + + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Empty(t, cfg.Evals, "a generate-only file is inert until init wires an eval") +} + +func TestValidate_Accepts(t *testing.T) { + require.NoError(t, loadFromString(t, sampleEvalConfig).Validate()) +} + +func TestValidate_Rejects(t *testing.T) { + const oneEval = "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + cases := []struct { + name string + body string + wantErr string + }{ + { + // The run path refuses a trace source that does not say whose + // conversations to read. Accepting it here would deploy a config + // that cannot run. + name: "trace source naming no agent", + body: "evals:\n - name: e\n source:\n type: traces\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.agent_name is required", + }, + { + // A model target names a deployment. The run refuses to filter + // spans by one, so accepting it here would deploy a config that + // cannot run -- which is what this check exists to prevent. + name: "trace source pointed at a model target", + body: "evals:\n - name: e\n source:\n type: traces\n" + + " target:\n type: model\n name: gpt-4o-mini\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.agent_name is required", + }, + { + name: "responses source listing no ids", + body: "evals:\n - name: e\n source:\n type: responses\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.response_ids is required", + }, + { + // A window bound the run path cannot parse is dropped by the + // service, which then grades a default seven days and says nothing. + name: "window bound that is not a time", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: yesterday\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "which is not a time", + }, + { + name: "window ending before it starts", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"2026-08-02T00:00:00Z\"\n end_time: \"2026-08-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "holds no traces", + }, + { + name: "window declared twice over", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"2026-08-01T00:00:00Z\"\n lookback_hours: 24\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "keep one", + }, + { + name: "lookback reaching forwards", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " lookback_hours: -24\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "how far back to look cannot be negative", + }, + { + // The bound exists to keep a typo from becoming a query over every + // trace ever recorded. Checked one past it, so raising the constant + // without meaning to fails here. + name: "lookback beyond what a window may cover", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " lookback_hours: 87601\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is beyond the 87600 hours a window can reach back", + }, + { + // Parses, then reads as "no bound" everywhere after, so the bound + // the file declared would be dropped from the request in silence. + name: "window bound at the zero time", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"0001-01-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not a time any traces were recorded at", + }, + { + // The wire drops a zero as readily as Go does, so an end bound at + // the epoch is the same silence one field over. + name: "end bound at the unix epoch", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " end_time: \"1970-01-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.end_time", + }, + { + name: "negative trace cap", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " max_traces: -5\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.max_traces is -5", + }, + { + // A responses source reads no traces, so a window on it bounds + // nothing and only looks as though it does. + name: "trace window on a responses source", + body: "evals:\n - name: e\n source:\n type: responses\n" + + " response_ids: [resp_1]\n lookback_hours: 24\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source declares lookback_hours, which a \"responses\" source does not read", + }, + { + name: "dataset without a name", + body: "datasets:\n - source: ./d.jsonl\n" + oneEval, + wantErr: "'name' is required", + }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n" + oneEval, + wantErr: "duplicate dataset name", + }, + { + name: "built-in declared in the catalog", + body: "evaluators:\n - name: builtin.relevance\n" + oneEval, + wantErr: "needs no catalog entry", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n" + oneEval, + wantErr: "cannot be set with `source`", + }, + { + name: "no evals", + body: "datasets:\n - name: d\n", + // A catalog with no eval is what `generate` leaves behind, so the + // error has to name the command that declares one. + wantErr: "azd ai eval init", + }, + { + name: "duplicate eval", + body: oneEval + " - name: e\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "duplicate eval name", + }, + { + name: "no evaluators", + body: "evals:\n - name: e\n evaluators: []\n", + wantErr: "at least one evaluator is required", + }, + { + name: "dataset and source both declared", + body: "datasets:\n - name: d\nevals:\n - name: e\n dataset: d\n" + + " source:\n type: traces\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "declare one", + }, + { + name: "dataset not in the catalog", + body: "evals:\n - name: e\n dataset: missing\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not in the datasets catalog", + }, + { + name: "evaluator not in the catalog", + body: "evals:\n - name: e\n evaluators:\n - evaluator: quality\n", + wantErr: "not in the evaluators catalog", + }, + { + name: "duplicate criterion", + body: "evals:\n - name: e\n evaluators:\n" + + " - evaluator: builtin.relevance\n - evaluator: builtin.relevance\n", + wantErr: "duplicate criterion", + }, + { + name: "unsupported source type", + body: "evals:\n - name: e\n source:\n type: prompt\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is not supported", + }, + { + name: "unsupported target type", + body: "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + " target:\n type: prompt\n", + wantErr: "is not supported", + }, + { + name: "invalid evaluation level", + body: "evals:\n - name: e\n evaluation_level: sentence\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "evaluation_level", + }, + { + name: "two evals differing only by name", + body: "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "identical to", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := loadFromString(t, tc.body).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// Two evals that differ only in substance still resolve unambiguously by name, +// and the clash only matters to something about to deploy. Enforcing it on the +// way to a lookup stranded `run list --eval `, which had already been +// told which eval it meant, behind an error whose only escape was hand-editing +// the config. +func TestValidateForLookupAllowsWhatOnlyDeployingCannotTellApart(t *testing.T) { + body := "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n" + cfg := loadFromString(t, body) + + require.NoError(t, cfg.ValidateForLookup()) + require.Error(t, cfg.Validate(), "deploying still cannot tell the two apart") +} + +// Lookup still depends on names being unique, so that check stays. +func TestValidateForLookupStillRefusesADuplicateName(t *testing.T) { + body := "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: a\n evaluators:\n - evaluator: builtin.coherence\n" + + err := loadFromString(t, body).ValidateForLookup() + + require.Error(t, err) + require.Contains(t, err.Error(), "duplicate") +} + +// What an eval says about itself is not what resolving it by name depends on. +// One malformed entry used to stop `run list --eval ` listing +// anything, and the way out was to hand-edit a file the error did not mention. +// The run door checks the entry the run is actually about. +func TestValidateForLookupLeavesAnEvalsOwnDeclarationToDeploying(t *testing.T) { + cases := map[string]string{ + "a field the source does not read": "evals:\n - name: a\n source:\n" + + " type: traces\n agent_name: x\n max_turns: 3\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + "a mistyped source type": "evals:\n - name: a\n source:\n type: tracs\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + "an unusable window": "evals:\n - name: a\n source:\n type: traces\n" + + " agent_name: x\n lookback_hours: -1\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + } + + for name, body := range cases { + t.Run(name, func(t *testing.T) { + cfg := loadFromString(t, body) + + require.NoError(t, cfg.ValidateForLookup(), + "a lookup only needs the name to be present and unique") + require.Error(t, cfg.Validate(), "deploying it is another matter") + }) + } +} + +// outputDir accepts a directory or an explicit file path. +func TestArtifactPath(t *testing.T) { + cases := []struct { + name string + outputDir string + resource string + ext string + want string + }{ + {"directory derives the file name", "datasets", "support-golden", ".jsonl", + filepath.Join("base", "datasets", "support-golden.jsonl")}, + {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", + filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, + {"empty outputDir falls back to the base", "", "support-quality", ".json", + filepath.Join("base", "support-quality.json")}, + {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", + filepath.Join("base", "generated", "rubrics", "quality.yaml")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ArtifactPath("base", tc.outputDir, tc.resource, tc.ext)) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go new file mode 100644 index 00000000000..1eb481bf18c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// FingerprintGroup is covered in service_target_eval_test.go. These cover the +// file hash and the environment key it is stored under, which nothing did. + +// A fingerprint is compared against the one recorded at the last deploy, so +// identical content must hash identically and a single changed byte must not. +func TestFingerprint(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.jsonl") + b := filepath.Join(dir, "b.jsonl") + require.NoError(t, os.WriteFile(a, []byte(`{"query":"hi"}`), 0o600)) + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hi"}`), 0o600)) + + sumA, err := Fingerprint(a) + require.NoError(t, err) + sumB, err := Fingerprint(b) + require.NoError(t, err) + + assert.Equal(t, sumA, sumB, "same content, same fingerprint") + assert.Len(t, sumA, 64, "sha-256 as hex") + + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hI"}`), 0o600)) + sumB, err = Fingerprint(b) + require.NoError(t, err) + assert.NotEqual(t, sumA, sumB, "one changed byte has to show") +} + +// A missing file names itself, because the usual cause is a catalog entry +// pointing at something that was moved or never generated. +func TestFingerprint_MissingFileNamesIt(t *testing.T) { + _, err := Fingerprint(filepath.Join(t.TempDir(), "gone.jsonl")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "gone.jsonl") +} + +// The key goes into an azd environment file, which accepts only uppercase +// letters, digits and underscores. A name that reached it unmapped would +// produce a key azd cannot round-trip, and the artifact would look changed on +// every deploy. +func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { + tests := []struct { + kind, name, readable string + }{ + {"dataset", "support-regression", "DATASET_SUPPORT_REGRESSION"}, + {"evaluator", "quality.v2", "EVALUATOR_QUALITY_V2"}, + {"dataset", "Mixed Case Name", "DATASET_MIXED_CASE_NAME"}, + // One rune maps to one underscore, so a multi-byte character does not + // widen the key. + {"eval", "unicode-caf\u00e9", "EVAL_UNICODE_CAF_"}, + } + + for _, tt := range tests { + t.Run(tt.readable, func(t *testing.T) { + key := FingerprintKey(tt.kind, tt.name) + + assert.True(t, + strings.HasPrefix(key, EnvKeyFingerprintPrefix+tt.readable+"_"), + "the key stays readable: %q", key) + for _, r := range strings.TrimPrefix(key, EnvKeyFingerprintPrefix) { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } + }) + } +} + +// Two artifacts of different kinds can share a name, and they must not share a +// key — one would overwrite the other's recorded fingerprint. +func TestFingerprintKey_KindSeparatesTheNamespaces(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset", "quality"), + FingerprintKey("evaluator", "quality")) +} + +// The readable half of the key maps every character outside [A-Z0-9] to an +// underscore, so these names are indistinguishable in it. Sharing a key means +// sharing a recorded fingerprint, version and id: both artifacts then look +// changed on every deploy and republish forever. +func TestFingerprintKey_NamesThatSanitizeAlikeStillDiffer(t *testing.T) { + collidingNames := []string{ + "quality-a", + "quality_a", + "quality a", + "quality.a", + "quality/a", + "quality\u00e9a", + "qualityXa", + } + + seen := make(map[string]string, len(collidingNames)) + for _, name := range collidingNames { + key := FingerprintKey("evaluator", name) + if previous, clash := seen[key]; clash { + t.Fatalf("%q and %q share the key %q", previous, name, key) + } + seen[key] = name + } +} + +// The digest covers the kind and the name separately, so moving a character +// across the boundary is not the same artifact. +func TestFingerprintKey_TheKindBoundaryIsNotAmbiguous(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset_a", "b"), + FingerprintKey("dataset", "a_b")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go new file mode 100644 index 00000000000..2aa6c78def5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The optimize metadata's instruction_file pointer is only as trustworthy as +// the checkout it was read from. Left unchecked, cloning a repository and +// running generate would read a named local file and send it on as agent +// instructions. +func TestInstructionPointerCannotLeaveTheProject(t *testing.T) { + root := filepath.Join(string(filepath.Separator), "work", "proj") + + inside := []string{ + filepath.Join(root, "instructions.md"), + filepath.Join(root, "src", "agent", ".agent_configs", "baseline", "i.md"), + filepath.Join(root, "a", "..", "b", "i.md"), + root, + } + for _, p := range inside { + assert.Truef(t, withinDir(root, p), "%q is inside the project", p) + } + + outside := []string{ + filepath.Join(root, "..", "other", "secrets.txt"), + filepath.Join(root, "..", "..", "etc", "passwd"), + filepath.Join(string(filepath.Separator), "etc", "passwd"), + filepath.Join(root+"-sibling", "i.md"), // prefix match, different directory + } + for _, p := range outside { + assert.Falsef(t, withinDir(root, p), "%q is outside the project", p) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go new file mode 100644 index 00000000000..218f5c159bb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// resolveMaxSamples reads anything not above zero as "no cap", so a negative +// max_samples used to send the WHOLE dataset to a run that is billed per row -- +// the opposite of what a cap asks for, and with nothing said about it. +func TestNegativeMaxSamplesIsRefused(t *testing.T) { + cfg := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{{ + Name: "support-quality", + Dataset: "golden", + EvaluationLevel: "turn", + MaxSamples: -1, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }}, + } + + err := cfg.Validate() + + require.Error(t, err) + assert.Contains(t, err.Error(), "max_samples") + assert.Contains(t, err.Error(), "support-quality", "the eval that carries it") + assert.Contains(t, err.Error(), "-1", "and the value that was rejected") +} + +// Zero is how a config says "send every row", and has to keep working. +func TestUnsetMaxSamplesIsStillAllowed(t *testing.T) { + cfg := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{{ + Name: "support-quality", + Dataset: "golden", + EvaluationLevel: "turn", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }}, + } + require.NoError(t, cfg.Validate()) + + cfg.Evals[0].MaxSamples = 25 + assert.NoError(t, cfg.Validate(), "a positive cap is the ordinary case") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go new file mode 100644 index 00000000000..dbb6a249284 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "bytes" +) + +// utf8BOM is what Windows editors and PowerShell's Set-Content write ahead of +// otherwise valid UTF-8. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// ReadFileNoBOM reads a file the user may have edited by hand, without the byte +// order mark a Windows editor puts in front of it. +// +// Neither encoding/json nor yaml.v3 skips one, and the error they raise names a +// character rather than the cause: "invalid character 'ï' looking for beginning +// of value" is not something a developer can act on. +func ReadFileNoBOM(path string) ([]byte, error) { + data, err := readFileOverContention(path) + if err != nil { + return nil, err + } + return bytes.TrimPrefix(data, utf8BOM), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go new file mode 100644 index 00000000000..07ba4497226 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Notepad, VS Code on Windows and PowerShell's Set-Content all write a BOM. +// Neither encoding/json nor yaml.v3 skips one, and what a developer sees is +// "invalid character 'ï' looking for beginning of value" — which names a +// character, not the cause. Hit for real while editing a rubric by hand. +func TestReadFileNoBOM_StripsTheWindowsByteOrderMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + body := append([]byte{0xEF, 0xBB, 0xBF}, []byte(`{"dimensions":[]}`)...) + require.NoError(t, os.WriteFile(path, body, 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, `{"dimensions":[]}`, string(data)) +} + +// A file without one is returned byte for byte. +func TestReadFileNoBOM_LeavesOrdinaryContentAlone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"dimensions":[]}`), 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, `{"dimensions":[]}`, string(data)) +} + +// Only a leading mark is a BOM. The same bytes inside the content are content. +func TestReadFileNoBOM_OnlyStripsALeadingMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + body := []byte("{\"note\":\"\uFEFF inside\"}") + require.NoError(t, os.WriteFile(path, body, 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, string(body), string(data)) +} + +// A configuration saved by a Windows editor has to load, or every command that +// reads it fails at once. +func TestLoadEvalConfig_AcceptsAByteOrderMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + body := append([]byte{0xEF, 0xBB, 0xBF}, + []byte("datasets:\n - name: d\n source: ./d.jsonl\n")...) + require.NoError(t, os.WriteFile(path, body, 0o600)) + + cfg, err := LoadEvalConfig(path) + + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "d", cfg.Datasets[0].Name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go new file mode 100644 index 00000000000..7f1966391aa --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// A declared source is relative to the configuration, but a user may write an +// absolute one. `eval create` used to join it unconditionally, producing +// evals/C:/data/rows.jsonl, while `azd up` resolved it correctly -- so the same +// file worked or failed depending on which command published it. +func TestResolveSourceLeavesAnAbsolutePathAlone(t *testing.T) { + abs := filepath.Join(string(filepath.Separator), "data", "rows.jsonl") + if filepath.VolumeName(`C:\`) != "" { + abs = `C:\data\rows.jsonl` + } + + assert.Equal(t, abs, ResolveSource("evals", abs), + "an absolute source is already where it says it is") + assert.Equal(t, filepath.Join("evals", "datasets", "rows.jsonl"), + ResolveSource("evals", "./datasets/rows.jsonl"), + "a relative source hangs off the configuration's directory") + assert.Empty(t, ResolveSource("evals", ""), + "nothing declared stays nothing, rather than becoming the directory") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go new file mode 100644 index 00000000000..5d63072957d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import "azureaieval/internal/messages" + +// ValidateRunnable refuses a declaration no run could carry out. +// +// One definition of what an eval has to say about itself, called on the way to +// deploying it and again when a run is built. Resolving an eval by name does +// not check any of this -- a lookup depends only on the name -- and a run +// reached by id has no declaration to check, so the run door is the first +// refusal as often as the config door is. Two hand-written copies drifted apart +// every time, on whichever axis was not tested at both ends. +// +// The errors carry no prefix. The caller says whether it has an index to name. +// +// What is not here is what the rest of the file, or the service, has to decide: +// whether a dataset or an evaluator is in its catalog, whether two evals are +// the same in substance, and what a published evaluator requires. The evaluator +// checks stay at the create door because they constrain what is created rather +// than what a run sends. +func ValidateRunnable(eval *Eval) error { + if eval == nil { + return messages.NoEvalToValidate() + } + // Two answers to where rows come from, and the file does not say which was + // meant. Refused rather than ranked: settling it by which field is read + // first sends a request that succeeds and grades the other one. + if eval.Dataset != "" && eval.Source != nil { + return messages.DatasetAndSourceDeclareTheSameThing() + } + if eval.MaxSamples < 0 { + return messages.MaxSamplesNegative(eval.MaxSamples) + } + + // The target is checked before the source, because the trace rule below + // reads the target: without this, an eval with an unusable target is told + // to name an agent on it, and told on the next run that the target it was + // sent to name is a kind nothing can invoke. + if eval.Target != nil { + if eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return messages.TargetTypeNotSupported(eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + // A target with no name is scored as though nothing were invoked, + // which is a different evaluation from the one that was written down. + if eval.Target.Name == "" { + return messages.TargetNameMissing() + } + } + + if eval.Source != nil { + switch eval.Source.Type { + case SourceTypeTraces: + if TraceAgentName(eval.Source, eval.Target) == "" { + // A model target is the one case where a target is present and + // still no answer. Saying "or declare an agent target.name" + // there reads as an invitation to relabel the deployment, which + // produces a filter that matches no spans and reports nothing. + if eval.Target != nil && eval.Target.Type == TargetTypeModel { + return messages.TraceSourceCannotReadAModelTarget(eval.Target.Name) + } + return messages.TraceSourceNeedsAnAgent() + } + case SourceTypeResponses: + if len(eval.Source.ResponseIDs) == 0 { + return messages.ResponsesSourceNeedsResponseIDs() + } + case "": + return messages.SourceTypeMissing() + default: + return messages.SourceTypeNotSupported( + eval.Source.Type, SourceTypeTraces, SourceTypeResponses) + } + if _, _, err := ValidateSource(eval.Source); err != nil { + return err + } + } + + switch eval.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + // Sent as run metadata, and anything that is not "conversation" is + // read as turn-shaped, so a value nothing knows about grades the run at + // a granularity the file did not ask for. + return messages.EvaluationLevelNotSupported( + eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go new file mode 100644 index 00000000000..8003771f91f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// One definition, two reporters. The configuration and the request builder both +// have to decide whether a declaration can run, and every round they decided it +// separately they drifted: one accepted what the other refused, and which rules +// applied depended on which door the eval came through. +// +// The errors carry no prefix of their own -- the caller adds what it knows -- +// so the rule can be stated once and reported from either side. +func TestValidateRunnable_RefusesWhatNoRunCouldCarryOut(t *testing.T) { + dataset := func(e Eval) Eval { e.Name, e.Dataset = "e", "d"; return e } + + cases := []struct { + name string + eval Eval + wantErr string + }{ + { + "rows from two places", + Eval{Dataset: "d", Source: &SourceDecl{Type: SourceTypeTraces, AgentName: "a"}}, + "declare one", + }, + {"a negative cap", dataset(Eval{MaxSamples: -1}), "max_samples cannot be negative"}, + { + "a target naming nothing", + dataset(Eval{Target: &Target{Type: TargetTypeAgent}}), + "target.name is required", + }, + { + "a target nothing can invoke", + dataset(Eval{Target: &Target{Type: "prompt", Name: "x"}}), + `target.type "prompt" is not supported`, + }, + { + "a source that does not say what it reads", + Eval{Name: "e", Source: &SourceDecl{}}, + "source.type is required", + }, + { + "a source nothing can read", + Eval{Name: "e", Source: &SourceDecl{Type: "trace"}}, + `source.type "trace" is not supported`, + }, + { + "a trace source naming no agent", + Eval{Name: "e", Source: &SourceDecl{Type: SourceTypeTraces}}, + "source.agent_name is required", + }, + { + "a responses source listing nothing", + Eval{Name: "e", Source: &SourceDecl{Type: SourceTypeResponses}}, + "source.response_ids is required", + }, + { + "a window the source cannot use", + Eval{Name: "e", Source: &SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", LookbackHours: -1, + }}, + "cannot be negative", + }, + { + // Sent as run metadata, and anything that is not "conversation" is + // read as turn-shaped, so a value nothing knows about grades the run + // at a granularity the file did not ask for. + "a granularity nothing scores at", + dataset(Eval{EvaluationLevel: "sentence"}), + `evaluation_level "sentence" is invalid`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateRunnable(&tc.eval) + + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + // The caller says where the error came from. A prefix here would + // be repeated by one door and wrong at the other. + assert.NotContains(t, err.Error(), "evals[") + assert.NotContains(t, err.Error(), `eval "`) + }) + } +} + +// A trace eval pointed at a model deployment is the one case where a target is +// present and still answers nothing. The general advice reads as an invitation +// to relabel the deployment as an agent, which produces a filter matching no +// spans and a run that reports nothing. +func TestValidateRunnable_SaysWhyAModelTargetIsNotAnAgent(t *testing.T) { + err := ValidateRunnable(&Eval{ + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: TargetTypeModel, Name: "gpt-4o-mini"}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "is a model deployment") + assert.NotContains(t, err.Error(), "declare an agent target.name", + "advice that leaves the eval wrong in a way nothing reports") +} + +// An eval wrong in two ways is told about the one that cannot be worked around, +// so following the advice does not lead straight back here. +func TestValidateRunnable_ReportsTheUnusableTargetBeforeTheRuleThatReadsIt(t *testing.T) { + err := ValidateRunnable(&Eval{ + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: "prompt"}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "is not supported") + assert.NotContains(t, err.Error(), "agent_name", + "naming an agent on a target nothing can invoke fixes nothing") +} + +// A declaration that says nothing contradictory passes, whichever shape it is. +func TestValidateRunnable_Accepts(t *testing.T) { + for name, eval := range map[string]Eval{ + "a dataset scored as it stands": {Name: "e", Dataset: "d"}, + "a dataset with an agent target": { + Name: "e", Dataset: "d", Target: &Target{Type: TargetTypeAgent, Name: "a"}, + }, + "a dataset with an untyped target": { + Name: "e", Dataset: "d", Target: &Target{Name: "a"}, + }, + "traces filtered by name": { + Name: "e", Source: &SourceDecl{Type: SourceTypeTraces, AgentName: "a"}, + }, + "traces named by the target": { + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: TargetTypeAgent, Name: "a"}, + }, + "stored responses": { + Name: "e", + Source: &SourceDecl{Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}}, + }, + } { + t.Run(name, func(t *testing.T) { + require.NoError(t, ValidateRunnable(&eval)) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go new file mode 100644 index 00000000000..73f59c69ff7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func serviceWith(t *testing.T, props map[string]any) *azdext.ServiceConfig { + t.Helper() + s, err := structpb.NewStruct(props) + require.NoError(t, err) + return &azdext.ServiceConfig{Name: "support-agent-evals", AdditionalProperties: s} +} + +// `azd up` reads the configuration through the service entry, not off disk, and +// that route used json.Unmarshal -- which drops unknown keys silently. So a +// misspelled key was named by `azd ai eval run` and ignored by `azd up`, and +// the setting the author thought they had wrote simply did not exist. +// +// Both routes now go through the same strict decoder. +func TestEvalConfigFromServiceRejectsAMistypedKey(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaulators": []any{}, // the typo `azd ai eval run` already catches + "evaluation_level": "turn", + }}, + }) + + _, err := EvalConfigFromService(svc, "") + + require.Error(t, err, "a key this extension does not know is a typo, on either route") + assert.Contains(t, err.Error(), "evaulators") + assert.Contains(t, err.Error(), "evaluators", "the near miss is what makes it actionable") +} + +// The keys the schema does know still decode, so the strictness did not close +// the door on the authoring style it is meant to serve. +func TestEvalConfigFromServiceAcceptsADeclaredConfig(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "datasets": []any{map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}}, + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "dataset": "golden", + "evaluation_level": "turn", + }}, + }) + + cfg, err := EvalConfigFromService(svc, "") + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, "support-agent-eval", cfg.Evals[0].Name) + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "golden", cfg.Datasets[0].Name) +} + +// `$ref` is a directive rather than configuration. ResolveFileRefs replaces it +// with the file's content, but it survives when resolution was skipped, and a +// strict decoder would then refuse a config for carrying the very thing that +// pointed at it. +func TestEvalConfigFromServiceIgnoresTheRefDirective(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "$ref": "./evals/azure.eval.yaml", + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaluation_level": "turn", + }}, + }) + + cfg, err := EvalConfigFromService(svc, "") + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go new file mode 100644 index 00000000000..66fe3626302 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -0,0 +1,391 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/azure/azure-dev/cli/azd/pkg/foundry" + "go.yaml.in/yaml/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +// EvalHost is the azure.yaml host this provider serves. +const EvalHost = "azure.ai.eval" + +// azd environment keys owned by this extension. +const ( + EnvKeyEvalID = "EVAL_ID" + EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" + EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) + +// Reconciler applies the eval configuration to the service. It is satisfied by +// the command layer, which owns the data-plane clients. +type Reconciler interface { + // EnsureDataset registers a new dataset version when the local content + // changed, returning the resolved version and whether anything was written. + EnsureDataset(ctx context.Context, decl DatasetDecl, localPath string) (version string, changed bool, err error) + // EnsureEvaluator registers a new evaluator version when the definition + // differs from what the service already holds. + EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) + // EnsureEval creates the group when it is absent or its resolved + // evaluators or options changed, returning its id. datasetPath is the local + // dataset backing the group, or empty when it is already registered; it lets + // the reconciler bind criteria to the columns that actually exist. + EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, created bool, err error) +} + +// EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns +// ordering across services through `uses:`; this provider owns only the order +// within the eval service itself. +type EvalServiceTargetProvider struct { + azdClient *azdext.AzdClient + newReconciler func(ctx context.Context) (Reconciler, error) + + serviceConfig *azdext.ServiceConfig +} + +// NewEvalServiceTargetProvider builds the provider. The reconciler is supplied +// lazily so the data-plane clients are only created when a deploy actually runs. +func NewEvalServiceTargetProvider( + azdClient *azdext.AzdClient, + newReconciler func(ctx context.Context) (Reconciler, error), +) *EvalServiceTargetProvider { + return &EvalServiceTargetProvider{azdClient: azdClient, newReconciler: newReconciler} +} + +func (p *EvalServiceTargetProvider) Initialize( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) error { + p.serviceConfig = serviceConfig + return nil +} + +// Endpoints reports no endpoints: eval resources are not addressable. +func (p *EvalServiceTargetProvider) Endpoints( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + targetResource *azdext.TargetResource, +) ([]string, error) { + return nil, nil +} + +func (p *EvalServiceTargetProvider) GetTargetResource( + ctx context.Context, + subscriptionId string, + serviceConfig *azdext.ServiceConfig, + defaultResolver func() (*azdext.TargetResource, error), +) (*azdext.TargetResource, error) { + if defaultResolver != nil { + if target, err := defaultResolver(); err == nil { + return target, nil + } + } + // Eval resources live on the project data plane, so there is no ARM + // resource of our own to resolve. + return &azdext.TargetResource{SubscriptionId: subscriptionId}, nil +} + +// Package is a no-op: eval artifacts are plain files already on disk. +func (p *EvalServiceTargetProvider) Package( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + progress azdext.ProgressReporter, +) (*azdext.ServicePackageResult, error) { + return &azdext.ServicePackageResult{}, nil +} + +// Publish is a no-op: there is no artifact registry step for eval resources. +func (p *EvalServiceTargetProvider) Publish( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + publishOptions *azdext.PublishOptions, + progress azdext.ProgressReporter, +) (*azdext.ServicePublishResult, error) { + return &azdext.ServicePublishResult{}, nil +} + +// Deploy reconciles the eval configuration in a fixed order — datasets, then +// evaluators, then evals — because a group references the versions the +// first two resolve to. It fails fast; the next `azd up` resumes from wherever +// it stopped. +func (p *EvalServiceTargetProvider) Deploy( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + progress azdext.ProgressReporter, +) (*azdext.ServiceDeployResult, error) { + cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) + if err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, messages.EvalConfigInvalid(err) + } + + reconciler, err := p.newReconciler(ctx) + if err != nil { + return nil, err + } + + baseDir := serviceRelativeDir(serviceConfig) + + // 1. Datasets the configuration owns. Paths are kept so an eval that names + // one can derive its columns without reading the blob back. + // + // A declaration with no `source:` is included rather than skipped: it names + // a dataset that is already registered, and reconciling it is what confirms + // it is really there and settles which version a `version:` pin selected. + // Skipping it would leave a misspelled name to surface as a failed run. + datasetPaths := map[string]string{} + for _, decl := range cfg.Datasets { + report(progress, messages.ReconcilingDataset(decl.Name)) + localPath := ResolveSource(baseDir, decl.Source) + datasetPaths[decl.Name] = localPath + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + if err != nil { + return nil, messages.DatasetProblem(decl.Name, err) + } + report(progress, describeResult("dataset", decl.Name, version, changed)) + } + + // 2. Evaluators this configuration owns. Built-ins and already-registered + // ones need no publish. + for _, decl := range cfg.CustomEvaluators() { + report(progress, messages.ReconcilingEvaluator(decl.Name)) + localPath := ResolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) + if err != nil { + return nil, messages.EvaluatorProblem(decl.Name, err) + } + report(progress, describeResult("evaluator", decl.Name, version, changed)) + } + + // 3. The evals. An eval is recreated only when its own declaration changed: + // the comparison covers what the entry declares, not what its references + // resolve to. An evaluator tracking latest that publishes a new version + // leaves every eval that runs it alone, which is what keeps a rubric edit + // comparable against the runs before it. + for i := range cfg.Evals { + eval := cfg.Evals[i] + report(progress, messages.ReconcilingEval(eval.Name)) + id, created, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset]) + if err != nil { + return nil, messages.EvalProblem(eval.Name, err) + } + report(progress, describeEval(eval.Name, id, created)) + } + + return &azdext.ServiceDeployResult{}, nil +} + +// projectRoot is the directory `$ref` paths resolve against. It is the +// directory holding azure.yaml, which only azd can report. +func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { + if p.azdClient == nil { + return "" + } + resp, err := p.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + return resp.GetProject().GetPath() +} + +// describeResult reports whether a version was published or reused, so a +// no-op deploy is visibly a no-op. +func describeResult(kind, name, version string, changed bool) string { + if changed { + return messages.PublishedVersion(kind, name, version) + } + return messages.UnchangedAtVersion(kind, name, version) +} + +// describeEval keeps a deploy's eval line saying the same thing the direct +// command says. Reporting the id either way left a deploy unable to answer +// whether it published anything. +func describeEval(name, id string, created bool) string { + if created { + return messages.EvalCreatedProgress(name, id) + } + return messages.EvalUnchangedProgress(name, id) +} + +func report(progress azdext.ProgressReporter, message string) { + if progress != nil { + progress(message) + } +} + +// EvalConfigFromService reads the eval configuration carried inline on the +// service entry. azd captures unknown keys into AdditionalProperties and hands +// them to the extension untouched. +// +// azd core deliberately does not resolve `$ref` includes for extensions — it +// strips the ServiceConfig fields it owns and leaves `$ref` at the top of the +// map for the owning extension to resolve. Without this call a service written +// as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, +// because the config parses to an empty set of datasets and groups. +func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { + props := serviceProps(svc) + if props == nil || len(props.GetFields()) == 0 { + return nil, messages.ServiceCarriesNoConfig(svc.GetName()) + } + + values := props.AsMap() + if projectRoot != "" { + resolved, err := foundry.ResolveFileRefs(values, projectRoot) + if err != nil { + return nil, messages.ResolvingServiceRefs(err) + } + values = resolved + } + + // `$ref` is a directive, not configuration: ResolveFileRefs has already + // replaced it with the file's content. It only survives when resolution was + // skipped, and that config cannot deploy anyway. + delete(values, "$ref") + + // Decoded by the same strict reader the on-disk path uses, so `azd up` and + // `azd ai eval run` name a mistyped key identically instead of one + // explaining it and the other ignoring it. + raw, err := yaml.Marshal(values) + if err != nil { + return nil, messages.ReadingServiceConfig(err) + } + cfg, err := DecodeEvalConfig(raw, svc.GetName()) + if err != nil { + return nil, err + } + return cfg, nil +} + +// serviceProps prefers the inline properties, falling back to the nested +// config block. +func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { + if s := svc.GetAdditionalProperties(); s != nil && len(s.GetFields()) > 0 { + return s + } + return svc.GetConfig() +} + +// serviceRelativeDir returns the directory that `source:` paths resolve against. +// +// When the service is authored as `host:` + `$ref: ./evals/azure.yaml`, the +// paths inside that file are written relative to the file itself, so the +// include's own directory is the base. ResolveFileRefs inlines the content +// without rebasing paths, so the base has to be recovered from the `$ref` +// value before resolution. +func serviceRelativeDir(svc *azdext.ServiceConfig) string { + if svc == nil { + return "." + } + if props := serviceProps(svc); props != nil { + if ref, ok := props.AsMap()["$ref"].(string); ok && ref != "" { + if dir := filepath.Dir(filepath.FromSlash(ref)); dir != "" { + return dir + } + } + } + if p := svc.GetRelativePath(); p != "" { + return p + } + return "." +} + +// ResolveSource joins a declared source against the directory holding the +// configuration, leaving absolute paths and empty values alone. +// +// Exported because `eval create` resolves the same declarations as `azd up` +// and had grown its own copy that joined unconditionally, so an absolute +// source came out as evals/C:/data/rows.jsonl there while `azd up` handled it. +// One resolver is what stops the two drifting again. +func ResolveSource(baseDir, source string) string { + if source == "" { + return "" + } + if filepath.IsAbs(source) { + return source + } + return filepath.Join(baseDir, source) +} + +// Fingerprint hashes a local artifact so a later deploy can tell whether the +// content changed without downloading anything from the service. +// +// The dataset API returns no content hash or etag, so comparing against the +// service would mean downloading the blob on every deploy. Every artifact this +// applies to — a dataset, a rubric, an evaluator script — is a single file. +func Fingerprint(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", messages.Hashing(path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintGroup hashes an eval's own declaration. +// +// Change detection on upstream artifacts is not sufficient: editing a group's +// evaluators, target, or options changes what the group means, and groups are +// immutable, so the group has to be recreated even when the dataset and +// evaluators are untouched. Without this a retargeted group keeps running +// against the old definition. +func FingerprintGroup(group Eval) (string, error) { + // Only substance is hashed. The id is server-assigned; name and description + // are what UpdateEvalParametersBody reaches, so an edit confined to them is + // pushed in place and must not cost the eval its id and its run history. + // Everything else — dataset, source, evaluators, target, level — is fixed at + // creation, so a change there is a new eval. + name := group.Name + group.ID = "" + group.Name = "" + group.Description = "" + + data, err := json.Marshal(group) + if err != nil { + return "", messages.HashingEval(name, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintKey is the azd environment key holding an artifact's fingerprint. +// +// The readable half is lossy: everything outside [A-Z0-9] becomes an +// underscore, so `quality-a`, `quality_a` and `quality a` all sanitize alike, +// as does any pair of names differing only outside ASCII. Two artifacts sharing +// a key overwrite each other's recorded fingerprint, version and id, which +// makes every deploy republish both. The trailing digest keeps them apart. +func FingerprintKey(kind, name string) string { + readable := strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r >= 'a' && r <= 'z': + return r - 32 + default: + return '_' + } + }, kind+"_"+name) + + sum := sha256.Sum256([]byte(kind + "\x00" + name)) + return EnvKeyFingerprintPrefix + readable + "_" + strings.ToUpper(hex.EncodeToString(sum[:4])) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go new file mode 100644 index 00000000000..e20ecd49009 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func propsFrom(t *testing.T, values map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(values) + require.NoError(t, err) + return s +} + +// A service authored as `host:` + `$ref: ./evals/azure.yaml` has its relative +// source paths written against the included file, not the project root. +// ResolveFileRefs inlines the content without rebasing them, so the base has to +// come from the $ref value. +func TestServiceRelativeDirUsesRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("evals"), serviceRelativeDir(svc)) +} + +// A nested include keeps its own directory. +func TestServiceRelativeDirUsesNestedRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./config/evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("config/evals"), serviceRelativeDir(svc)) +} + +// Without a $ref the service's own relative path is the base. +func TestServiceRelativeDirFallsBackToRelativePath(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + RelativePath: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{}, + }), + } + require.Equal(t, "evals", serviceRelativeDir(svc)) +} + +// With neither, sources resolve against the project root. +func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { + require.Equal(t, ".", serviceRelativeDir(&azdext.ServiceConfig{Name: "evals"})) + require.Equal(t, ".", serviceRelativeDir(nil)) +} + +// An inline config still parses when no project root is available to resolve +// includes against. +func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "support-agent-evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evals": []any{ + map[string]any{ + "name": "support-agent-smoke", + "dataset": "golden", + "evaluators": []any{ + map[string]any{"evaluator": "builtin.task_adherence"}, + }, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, + }), + } + + cfg, err := EvalConfigFromService(svc, "") + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + + // One service covers every eval in the file it pulled in, so the eval is + // selected by its own name rather than by the service key. + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "golden", eval.Dataset) + require.Len(t, eval.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", eval.Evaluators[0].Evaluator) + require.Equal(t, "my-agent", eval.Target.Name) +} + +func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { + _, err := EvalConfigFromService(&azdext.ServiceConfig{Name: "evals"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no eval configuration") +} + +// Evals are immutable, so a change to an eval's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting an +// eval at a different agent leaves the dataset and evaluators untouched. +func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { + base := Eval{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + EvaluationLevel: EvaluationLevelTurn, + } + + original, err := FingerprintGroup(base) + require.NoError(t, err) + + same, err := FingerprintGroup(base) + require.NoError(t, err) + require.Equal(t, original, same, "an unchanged eval must keep its fingerprint") + + cases := map[string]func(g *Eval){ + "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *Eval) { + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + }, + "judge deployment": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, + }} + }, + "version pin": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", Version: "2", + }} + }, + "evaluation level": func(g *Eval) { g.EvaluationLevel = EvaluationLevelConversation }, + "dataset": func(g *Eval) { g.Dataset = "other" }, + "source": func(g *Eval) { + g.Dataset = "" + g.Source = &SourceDecl{Type: SourceTypeTraces, AgentName: "agent-a"} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + changed := base + changed.Evaluators = append(evalcore.EvaluatorList(nil), base.Evaluators...) + mutate(&changed) + + digest, err := FingerprintGroup(changed) + require.NoError(t, err) + require.NotEqual(t, original, digest, "changing %s must change the fingerprint", name) + }) + } +} + +// The fingerprint covers substance only. The id is server-assigned, and name +// and description are what UpdateEvalParametersBody reaches — an edit confined +// to those is pushed in place, so it must not fork the run history. +func TestFingerprintGroupIgnoresIdNameAndDescription(t *testing.T) { + base := Eval{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + original, err := FingerprintGroup(base) + require.NoError(t, err) + + noisy := base + noisy.ID = "eval_abc123" + noisy.Name = "quality-renamed" + noisy.Description = "reworded" + + digest, err := FingerprintGroup(noisy) + require.NoError(t, err) + require.Equal(t, original, digest) +} + +// Editing one eval must not recreate its siblings: the unit compared is the +// eval's own subtree, never the file. +func TestFingerprintGroupIsScopedToOneEval(t *testing.T) { + gate := Eval{ + Name: "support-agent-gate", + Dataset: "prod-golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + regression := Eval{ + Name: "support-agent-regression-eval", + Dataset: "support-agent-regression", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + + before, err := FingerprintGroup(regression) + require.NoError(t, err) + + gate.Evaluators = append(gate.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + + after, err := FingerprintGroup(regression) + require.NoError(t, err) + require.Equal(t, before, after, "editing a sibling must leave this eval alone") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go new file mode 100644 index 00000000000..f243b8a3980 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "time" + + "azureaieval/internal/messages" +) + +// MaxLookbackHours bounds `lookback_hours` at ten years. +// +// Ten years is a policy bound, not an arithmetic one: the hours become a +// time.Duration in nanoseconds, which does not overflow until about 2,562,047 +// hours. The tighter bound is here because a lookback in that range is a typo +// rather than a window, and the run it produces is expensive and empty. +const MaxLookbackHours = 24 * 365 * 10 + +// TraceAgentName says whose conversations a trace eval reads. +// +// `agent_name` is the filter; an eval that leaves it off and names a target +// still means "this agent's traces". A model target is not an agent: it names a +// deployment, and filtering spans by a deployment name matches nothing, so the +// run comes back empty with no reason given. `target.type` is optional, and an +// untyped target is read as an agent, which is how the dataset path reads one. +// +// One definition, because the configuration check and the request builder both +// have to answer this and used to answer it separately: the config accepted a +// model target and every run of it then failed. +func TraceAgentName(source *SourceDecl, target *Target) string { + if source != nil && source.AgentName != "" { + return source.AgentName + } + if target == nil || target.Type == TargetTypeModel { + return "" + } + return target.Name +} + +// ValidateSource checks a source declaration and resolves the window it names. +// +// One definition of what a source may say, called by the configuration check +// and again when the request is built. Two copies drifted apart on every axis +// they were not both tested on, so which rules applied depended on which door +// the eval came through. +// +// The window is returned as well as checked, because a rule about a window can +// only be stated once the window is known, and the caller that sends the +// request needs the same bounds the caller that validated it saw. +// +// A zero start or end means unbounded at that end. +func ValidateSource(source *SourceDecl) (start, end time.Time, err error) { + if source == nil { + return time.Time{}, time.Time{}, nil + } + if err := validateSourceFields(source); err != nil { + return time.Time{}, time.Time{}, err + } + return resolveTraceWindow(source) +} + +// validateSourceFields refuses fields the declared source type does not read. +// +// A field that is quietly ignored is how a file comes to say something it does +// not do: a `lookback_hours` under `type: responses` looks like it bounds the +// run and never has, and nothing about the run it produces says otherwise. +func validateSourceFields(source *SourceDecl) error { + var inert []string + switch source.Type { + case SourceTypeTraces: + inert = namesOfSet( + sourceField{"response_ids", len(source.ResponseIDs) > 0}, + sourceField{"max_turns", source.MaxTurns != 0}, + ) + case SourceTypeResponses: + if source.MaxTurns < 0 { + return messages.MaxTurnsUnusable(source.MaxTurns) + } + inert = namesOfSet( + sourceField{"start_time", source.StartTime != ""}, + sourceField{"end_time", source.EndTime != ""}, + sourceField{"lookback_hours", source.LookbackHours != 0}, + sourceField{"max_traces", source.MaxTraces != 0}, + sourceField{"agent_name", source.AgentName != ""}, + sourceField{"agent_version", source.AgentVersion != ""}, + ) + default: + // An unsupported type is reported by the caller, which knows how to + // name the eval it came from and which types there are. + return nil + } + if len(inert) == 0 { + return nil + } + return messages.SourceFieldsNotRead(source.Type, inert) +} + +type sourceField struct { + name string + set bool +} + +func namesOfSet(fields ...sourceField) []string { + var names []string + for _, f := range fields { + if f.set { + names = append(names, f.name) + } + } + return names +} + +// resolveTraceWindow reads the span of traces an eval grades. +// +// The sole enforcement point for the sign and size of a lookback, the sign of +// max_traces, a window declared twice over, and the pre-epoch rule for both a +// written bound and a derived one. A new rule about the window belongs here. +func resolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { + // Parsed first, so a file that is wrong in two ways names the value that + // cannot be read at all rather than the pair it also got wrong. + start, err = traceBound("start_time", source.StartTime) + if err != nil { + return time.Time{}, time.Time{}, err + } + end, err = traceBound("end_time", source.EndTime) + if err != nil { + return time.Time{}, time.Time{}, err + } + + if source.LookbackHours < 0 { + return time.Time{}, time.Time{}, messages.NegativeLookbackHours(source.LookbackHours) + } + if source.LookbackHours > MaxLookbackHours { + return time.Time{}, time.Time{}, messages.LookbackTooLarge(source.LookbackHours, MaxLookbackHours) + } + if source.MaxTraces < 0 { + return time.Time{}, time.Time{}, messages.MaxTracesUnusable(source.MaxTraces) + } + // Two ways of saying where the window opens, and the file cannot say which + // was meant. Every other contradictory pair here is refused rather than + // ranked. + if source.StartTime != "" && source.LookbackHours != 0 { + return time.Time{}, time.Time{}, messages.TraceWindowOverSpecified() + } + + // The lookback is measured back from where the window closes, which is now + // when nothing closed it. Measuring from now regardless made the window a + // function of the clock: `lookback_hours` beside an `end_time` validated + // today and failed tomorrow, with the file unchanged. + if start.IsZero() && source.LookbackHours > 0 { + from := end + if from.IsZero() { + from = time.Now() + } + start = from.Add(-time.Duration(source.LookbackHours) * time.Hour) + // Held to the same rule as a written bound. A lookback long enough to + // reach past the epoch lands on a start the wire then drops, which is + // the silence the rule exists to break however the bound was reached. + if start.Unix() <= 0 { + return time.Time{}, time.Time{}, messages.LookbackReachesTooFarBack(source.LookbackHours) + } + return start, end, nil + } + + if !start.IsZero() && !end.IsZero() && !end.After(start) { + return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts( + source.StartTime, source.EndTime) + } + return start, end, nil +} + +// traceBound reads one end of the window. +// +// A bound at or before the Unix epoch is refused. Exactly zero has to be +// refused because zero is what every layer below reads as "no bound" -- Go's +// zero time here, and an omitted field on the wire -- so a bound that lands on +// it would be dropped from the request without a word. The rest of the +// pre-1970 half-line is refused with it because no trace was recorded there, +// and one rule about the whole span is easier to state than a hole in it. +func traceBound(name, value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, messages.TraceWindowNotATime(name, value) + } + if parsed.Unix() <= 0 { + return time.Time{}, messages.TraceWindowBoundUnusable(name, value) + } + return parsed, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go new file mode 100644 index 00000000000..122dec11960 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A lookback beside an end_time used to be measured from now, which made the +// same file valid today and invalid tomorrow with nothing edited: once now +// minus the lookback drifted past the end, the window was empty for good. +// Measuring back from where the window closes takes the clock out of it. +func TestValidateSource_LookbackMeasuresBackFromTheEnd(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{ + Type: SourceTypeTraces, + AgentName: "a", + LookbackHours: 24, + EndTime: "2020-01-01T00:00:00Z", + }) + + require.NoError(t, err) + assert.Equal(t, time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), end.UTC()) + assert.Equal(t, time.Date(2019, 12, 31, 0, 0, 0, 0, time.UTC), start.UTC()) + assert.True(t, end.After(start)) +} + +// With nothing closing the window, the lookback measures back from now. +func TestValidateSource_LookbackWithNoEndMeasuresBackFromNow(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", LookbackHours: 24, + }) + + require.NoError(t, err) + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), start.Unix(), 60) + assert.True(t, end.IsZero(), "an open end means up to now") +} + +// A source with no window at all is not an error: both ends open is what an +// eval that never mentioned a window means. +func TestValidateSource_OpenWindowIsFine(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{Type: SourceTypeTraces, AgentName: "a"}) + + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.True(t, end.IsZero()) + + start, end, err = ValidateSource(nil) + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.True(t, end.IsZero()) +} + +// One end bounded and the other open is a window, not an error: "everything +// since" and "everything up to" are both things an eval can mean. +func TestValidateSource_OneEndOpenIsAWindow(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{StartTime: "2026-08-01T00:00:00Z"}) + require.NoError(t, err) + assert.Equal(t, int64(1785542400), start.Unix()) + assert.True(t, end.IsZero()) + + start, end, err = ValidateSource(&SourceDecl{EndTime: "2026-08-02T00:00:00Z"}) + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.Equal(t, int64(1785628800), end.Unix()) +} + +// A lookback long enough to reach past the epoch lands on a start the wire +// drops, which is the same silence a written bound at the epoch is refused for. +// The bound is arrived at differently and has to be held to the same rule. +func TestValidateSource_RefusesALookbackPastTheEpoch(t *testing.T) { + _, _, err := ValidateSource(&SourceDecl{ + EndTime: "1970-01-01T01:00:00Z", LookbackHours: 1, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "before any trace was recorded") +} + +// A file wrong in two ways names the value that cannot be read at all, rather +// than a pair it also got wrong: fixing the pair would leave the unreadable +// value in place and send the reader round again. +func TestValidateSource_ReportsTheUnreadableValueFirst(t *testing.T) { + _, _, err := ValidateSource(&SourceDecl{StartTime: "yesterday", LookbackHours: -1}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "which is not a time") + assert.NotContains(t, err.Error(), "lookback_hours") +} + +// Fields the declared type never reads are refused rather than ignored: a +// lookback under a responses source looks like it bounds the run and never has. +func TestValidateSource_RefusesFieldsTheTypeDoesNotRead(t *testing.T) { + _, _, err := ValidateSource(&SourceDecl{ + Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, + LookbackHours: 24, AgentName: "a", + }) + require.Error(t, err) + // Named, because a reader with several set should not have to bisect. + assert.Contains(t, err.Error(), "lookback_hours, agent_name") + + _, _, err = ValidateSource(&SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", MaxTurns: 3, + }) + require.Error(t, err) + // One field reads "remove it", not "remove them". + assert.Contains(t, err.Error(), "source declares max_turns") + assert.Contains(t, err.Error(), "remove it") + + // max_traces is refused for its sign wherever it appears; max_turns is the + // same kind of value and was going out unchecked. + _, _, err = ValidateSource(&SourceDecl{ + Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, MaxTurns: -3, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "source.max_turns is -3") +} + +// Every rule, at the boundary rather than well past it, so a bound that is +// moved by one still fails. +func TestValidateSource_Refuses(t *testing.T) { + cases := []struct { + name string + source SourceDecl + wantErr string + }{ + { + name: "start that is not a time", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "yesterday"}, + wantErr: "source.start_time is \"yesterday\", which is not a time", + }, + { + name: "end that is not a time", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", EndTime: "tomorrow"}, + wantErr: "source.end_time is \"tomorrow\", which is not a time", + }, + { + name: "start at year one", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "0001-01-01T00:00:00Z"}, + wantErr: "not a time any traces were recorded at", + }, + { + // Parses, is not Go's zero time, and still serializes to a unix + // zero that omitempty drops from the request. + name: "start at the unix epoch", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "1970-01-01T00:00:00Z"}, + wantErr: "not a time any traces were recorded at", + }, + { + name: "negative lookback", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", LookbackHours: -1}, + wantErr: "how far back to look cannot be negative", + }, + { + name: "lookback one past the bound", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", LookbackHours: MaxLookbackHours + 1}, + wantErr: "beyond the 87600 hours", + }, + { + name: "negative cap", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", MaxTraces: -1}, + wantErr: "source.max_traces is -1", + }, + { + name: "window declared twice over", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-01T00:00:00Z", LookbackHours: 1}, + wantErr: "keep one", + }, + { + name: "end before start", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + wantErr: "holds no traces", + }, + { + // An instant is not a window, and a run over it reads nothing. + name: "end equal to start", + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-01T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + wantErr: "holds no traces", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := ValidateSource(&tc.source) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// The bound is exactly on the line, so the check is `>` and not `>=`. Asserted +// through the resolver rather than against the constant: a comparison of the +// constant with its own definition cannot fail, and the multiplication that +// would overflow is constant-folded, so an overflowing value would stop the +// package compiling rather than fail a test. +func TestValidateSource_AcceptsTheLargestLookbackAllowed(t *testing.T) { + start, _, err := ValidateSource(&SourceDecl{LookbackHours: MaxLookbackHours}) + + require.NoError(t, err) + assert.True(t, start.Before(time.Now()), "the window has to open in the past") + assert.True(t, start.After(time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)), + "an overflowed duration lands centuries away, not ten years") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go new file mode 100644 index 00000000000..e8c9dc47cf4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A typo is named wherever it appears, not only at the top level. +// +// yaml.Node.Decode does not inherit KnownFields from the decoder that reached +// it, so a misspelt key inside an evaluator entry was dropped in silence while +// the same misspelling one level up was reported. A pinned `verison` that does +// nothing is worse than one that is refused: the run grades against whatever +// version happens to be latest and reports success. +func TestUnknownKeysAreNamedAtEveryDepth(t *testing.T) { + cases := []struct { + where string + body string + key string + nearer string + line string + }{ + { + where: "top level of an eval", + body: "evals:\n - name: e1\n datasett: golden\n", + key: "datasett", + nearer: "dataset", + line: "line 3", + }, + { + where: "inside an evaluator entry", + body: "evals:\n - name: e1\n evaluators:\n - evaluator: builtin.x\n verison: \"3\"\n", + key: "verison", + nearer: "version", + line: "line 5", + }, + { + where: "inside a dataset declaration", + body: "datasets:\n - name: golden\n sourse: ./rows.jsonl\n", + key: "sourse", + nearer: "source", + line: "line 3", + }, + } + + for _, tc := range cases { + t.Run(tc.where, func(t *testing.T) { + _, err := DecodeEvalConfig([]byte(tc.body), "azure.eval.yaml") + require.Errorf(t, err, "%q was accepted in silence", tc.key) + + assert.Contains(t, err.Error(), tc.key, "the message has to name the key") + assert.Contains(t, err.Error(), tc.nearer, "and the key it was probably meant to be") + assert.Contains(t, err.Error(), tc.line, + "pointing at the line in the file, not inside an extracted fragment") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go new file mode 100644 index 00000000000..da6e50e25eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// An unquoted version is a number, and `azd up` reads the configuration through +// the service entry rather than off disk. That route arrives as protobuf, whose +// only numeric kind is a double, so `1` and `1.0` are the same value by the +// time this extension is handed it -- and it renders as "1", while reading the +// same file off disk gives "1.0". +// +// The spelling is destroyed before this code runs, so there is nothing here to +// recover it from: emitting "1.0" would break a config that meant 1, and +// rejecting the number would fail a file that `azd ai eval run` accepts. This +// pins the divergence so it is visible and cannot widen unnoticed. +// +// The fix available to a user is to quote it, which both routes preserve. +func TestNumericVersionLosesItsSpellingOnTheServiceRoute(t *testing.T) { + fromService := func(t *testing.T, version any) string { + t.Helper() + props, err := structpb.NewStruct(map[string]any{ + "evals": []any{map[string]any{ + "name": "support-quality", + "dataset": "golden", + "evaluation_level": "turn", + "evaluators": []any{map[string]any{ + "evaluator": "builtin.relevance", + "version": version, + }}, + }}, + }) + require.NoError(t, err) + + cfg, err := EvalConfigFromService( + &azdext.ServiceConfig{Name: "evals", AdditionalProperties: props}, "") + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + require.Len(t, cfg.Evals[0].Evaluators, 1) + return cfg.Evals[0].Evaluators[0].Version + } + + assert.Equal(t, "1", fromService(t, 1.0), + "1.0 and 1 are one number in protobuf, so the decimal cannot survive") + assert.Equal(t, "1.5", fromService(t, 1.5), + "a fractional part is not lost, only a trailing zero") + assert.Equal(t, "1.0", fromService(t, "1.0"), + "quoting is what carries the spelling through, and is the advice to give") +} + +// The same file read off disk keeps what the user wrote, which is the half of +// the divergence that behaves. +func TestQuotedAndUnquotedVersionsOffDisk(t *testing.T) { + cfg, err := DecodeEvalConfig([]byte(` +evals: + - name: support-quality + dataset: golden + evaluation_level: turn + evaluators: + - evaluator: builtin.relevance + version: 1.0 +`), "eval.yaml") + require.NoError(t, err) + assert.Equal(t, "1.0", cfg.Evals[0].Evaluators[0].Version, + "YAML hands a string field the scalar as written") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go new file mode 100644 index 00000000000..a0e038bd819 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package urlsafe renders URLs for logs and errors without their credentials. +// +// It exists because url.URL.Redacted looks like the safe choice and is not: it +// masks a userinfo password only, and leaves the query string untouched. A +// storage SAS carries its credential in the query as sig, so logging a SAS URI +// with Redacted writes a live credential to disk. +package urlsafe + +import ( + "errors" + "net/url" +) + +// URL renders a URL with its query and fragment removed, keeping the scheme, +// host and path so the log still says where the request went. +func URL(u *url.URL) string { + if u == nil { + return "" + } + safe := *u + safe.RawQuery = "" + safe.Fragment = "" + return safe.Redacted() +} + +// Error rebuilds a *url.Error without its request URL. http.Client.Do embeds +// the full URL in the error text, so a DNS, TLS, timeout or cancellation +// failure on a SAS-backed request would otherwise show the credential to the +// user. The original error is left unmodified. +func Error(err error) error { + urlError, ok := errors.AsType[*url.Error](err) + if !ok { + return err + } + safe := "" + if u, parseErr := url.Parse(urlError.URL); parseErr == nil { + safe = URL(u) + } + return &url.Error{Op: urlError.Op, URL: safe, Err: urlError.Err} +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go new file mode 100644 index 00000000000..21cf2f190b4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package urlsafe + +import ( + "errors" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sasSecret = "REDACT_ME_SECRET" + +// These tests pin the premise as well as the behaviour: url.URL.Redacted is the +// call that looks correct and leaks, so if someone reaches for it again the +// first assertion explains why they should not. +func TestURLDropsTheSASSignature(t *testing.T) { + raw := "https://acct.blob.core.windows.net/c/rows.jsonl?sv=2021-08-06&sig=" + sasSecret + u, err := url.Parse(raw) + require.NoError(t, err) + + assert.Contains(t, u.Redacted(), sasSecret, + "guards the premise: Redacted() alone leaks the signature") + + safe := URL(u) + assert.NotContains(t, safe, sasSecret, "the SAS signature must never reach a log") + assert.NotContains(t, safe, "sig=") + assert.Equal(t, "https://acct.blob.core.windows.net/c/rows.jsonl", safe, + "scheme, host and path stay, so the log still says where the request went") + assert.Equal(t, raw, u.String(), "the caller's URL is untouched and still usable") +} + +func TestURLHandlesNil(t *testing.T) { + assert.Equal(t, "", URL(nil)) +} + +func TestErrorStripsTheSASFromTransportFailures(t *testing.T) { + inner := errors.New("dial tcp: lookup failed") + original := &url.Error{ + Op: "Get", + URL: "https://acct.blob.core.windows.net/c/rows.jsonl?sig=" + sasSecret, + Err: inner, + } + + got := Error(original) + + assert.NotContains(t, got.Error(), sasSecret, + "a transport failure must not show the SAS to the user") + assert.Contains(t, got.Error(), "acct.blob.core.windows.net", + "the host stays so the message still says where it failed") + assert.ErrorIs(t, got, inner, "the cause stays unwrappable") + assert.Contains(t, original.URL, sasSecret, "the original error is not mutated") +} + +func TestErrorLeavesOtherErrorsAlone(t *testing.T) { + plain := errors.New("some other failure") + assert.Same(t, plain, Error(plain)) + assert.Nil(t, Error(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go new file mode 100644 index 00000000000..e7279d11fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package version + +var ( + // Populated at build time. + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/main.go b/cli/azd/extensions/azure.ai.evaluations/main.go new file mode 100644 index 00000000000..993d2e8816e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/main.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "azureaieval/internal/cmd" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +func main() { + azdext.Run(cmd.NewRootCommand()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go new file mode 100644 index 00000000000..94d7807a3c7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +type datasetSummary struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` +} + +const datasetRows = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +` + +// registeredDataset is a dataset with more than one version, which is what +// makes --version on show and --name on list worth asserting. +type registeredDataset struct { + Name string + // Versions are read back from each registration rather than assumed to + // start at 1: the server assigns them, and a test that hardcoded the + // numbering would be asserting its own guess. + Versions []string +} + +var ( + readOnlyDatasetOnce sync.Once + readOnlyDataset *registeredDataset +) + +// sharedDataset is registered once for the tests that only read it. Each +// registration uploads a blob, so redoing it per test buys nothing. +func sharedDataset(t *testing.T) *registeredDataset { + t.Helper() + readOnlyDatasetOnce.Do(func() { + readOnlyDataset = registerDataset(t, 2) + }) + require.NotNil(t, readOnlyDataset, "the shared dataset could not be registered") + return readOnlyDataset +} + +// registerDataset publishes a dataset and removes every version it created. +func registerDataset(t *testing.T, versions int) *registeredDataset { + t.Helper() + require.Positive(t, versions) + + path := filepath.Join(t.TempDir(), "golden.jsonl") + require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) + + ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} + for i := range versions { + // The first publish is a create; every later one is an update, which is + // the only difference between them. + verb := "update" + if i == 0 { + verb = "create" + } + r := requireSuccess(t, run(t, "dataset", verb, + ds.Name, "--from-file", path, "-o", "json")) + + var created datasetSummary + r.JSON(t, &created) + require.NotEmpty(t, created.Version, "the service assigns the version") + ds.Versions = append(ds.Versions, created.Version) + + version := created.Version + deferTeardown(func() { + runQuietly("dataset", "delete", ds.Name, "--version", version) + }) + } + require.Len(t, ds.Versions, versions) + require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], + "updating must advance the version rather than overwrite") + return ds +} + +func TestCLIDatasetList(t *testing.T) { + ds := sharedDataset(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name)) + // TYPE, not FORMAT. The service populates `type` (`uri_file`) and leaves + // `format` empty, so the column this once pinned was blank on every row. + for _, header := range []string{"NAME", "VERSION", "TYPE"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + // `versions list` is what makes the listing usable once a project holds more + // than a screenful: it narrows to one dataset's versions. + t.Run("versions list scopes to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name, "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.NotEmpty(t, listed) + + seen := map[string]bool{} + for _, v := range listed { + require.Equalf(t, ds.Name, v.Name, + "the listing must return only that dataset's versions; got %q", v.Name) + seen[v.Version] = true + } + for _, want := range ds.Versions { + require.Truef(t, seen[want], "version %s is missing from the listing", want) + } + }) + + // Unscoped, the listing is every dataset rather than every version, so the + // one just registered has to be in it. + t.Run("unscoped lists the project's datasets", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "-o", "json")) + var all []datasetSummary + r.JSON(t, &all) + require.NotEmpty(t, all) + + found := false + for _, d := range all { + if d.Name == ds.Name { + found = true + } + } + require.True(t, found, "a registered dataset must appear in the unscoped listing") + }) + + t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.Empty(t, listed) + }) +} + +func TestCLIDatasetShow(t *testing.T) { + ds := sharedDataset(t) + latest := ds.Versions[len(ds.Versions)-1] + + // Omitting the version means the latest, which is the only sensible + // default for a name that gains a version on every registration. + t.Run("defaults to the latest version", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", ds.Name, "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Name, shown.Name) + require.Equal(t, latest, shown.Version) + }) + + t.Run("version pins an earlier one", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", + ds.Name, "--version", ds.Versions[0], "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Versions[0], shown.Version) + require.NotEqual(t, latest, shown.Version) + }) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", ds.Name)) + // `show` reads one dataset, so it renders a detail view keyed by label + // rather than a one-row table with a header. The labels are what is + // under test; the column-header form belongs to `dataset list`. + for _, label := range []string{"Name", "Version", "URI"} { + require.Containsf(t, r.Stdout, label, "the detail view lost its %s line", label) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + t.Run("the name is required", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show")) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + + t.Run("an unknown dataset is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", "azdcli-no-such-dataset")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "azdcli-no-such-dataset") + }) + + t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", + ds.Name, "--version", "9999")) + require.Contains(t, r.Combined(), "9999") + require.Less(t, len(r.Combined()), 600, r.Combined()) + }) +} + +func TestCLIDatasetDelete(t *testing.T) { + t.Run("the name and version are both required", func(t *testing.T) { + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), + "accepts 1 arg") + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "whatever")).Combined(), + "--version is required") + }) + + // Deleting something that was never registered succeeds. The service + // treats DELETE as idempotent and answers 204 whatever the name, so the + // command reports a removal it did not perform — and the not-found branch + // in `dataset delete` cannot be reached this way. Asserted rather than + // wished away, because a caller scripting against the exit code is + // entitled to know it means "gone", not "was there and is now gone". + t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "delete", + "azdcli-no-such-dataset", "--version", "1")) + require.Contains(t, r.Stdout, "Deleted dataset") + + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + require.Empty(t, remaining, "nothing was there to delete in the first place") + }) + + // A successful delete answers 204 No Content, so asserting the exit code + // is what catches a client that reads an empty body as a failure and + // reports a removal it just performed as an error. + t.Run("one version is removed and the other survives", func(t *testing.T) { + ds := registerDataset(t, 2) + gone, kept := ds.Versions[0], ds.Versions[1] + + r := requireSuccess(t, run(t, "dataset", "delete", + ds.Name, "--version", gone)) + require.Contains(t, r.Stdout, "Deleted dataset") + require.Contains(t, r.Stdout, ds.Name) + + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + ds.Name, "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + + versions := map[string]bool{} + for _, v := range remaining { + versions[v.Version] = true + } + require.False(t, versions[gone], "the deleted version must leave the listing") + require.True(t, versions[kept], "deleting one version must not remove the others") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go new file mode 100644 index 00000000000..fbcab7ee126 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCLIEvaluatorListBuiltin is the cheapest proof the binary can reach the +// service on its own: no azd project, no config, just a flag. +func TestCLIEvaluatorListBuiltin(t *testing.T) { + r := requireSuccess(t, run(t, "evaluator", "list", "--builtin", "-o", "json")) + + var builtins []struct { + Name string `json:"name"` + EvaluatorType string `json:"evaluator_type"` + } + r.JSON(t, &builtins) + require.NotEmpty(t, builtins, "the project must expose built-in evaluators") + + for _, b := range builtins { + require.True(t, strings.HasPrefix(b.Name, "builtin."), + "--builtin must return only built-ins, got %q", b.Name) + } + + // The default rendering is a table, not JSON. A script reading stdout + // without -o json would otherwise silently parse a header row. + table := requireSuccess(t, run(t, "evaluator", "list", "--builtin")) + require.Contains(t, table.Stdout, "NAME") + require.Contains(t, table.Stdout, "VERSION") +} + +// TestCLIJSONListsAreBareArrays pins the envelope. +// +// The service wraps listings in {"value":[...]} or {"data":[...]} depending on +// the route. Leaking either would make every consumer special-case the +// command it came from, so the CLI unwraps them, and this is what says so. +func TestCLIJSONListsAreBareArrays(t *testing.T) { + for _, args := range [][]string{ + {"evaluator", "list", "--builtin", "-o", "json"}, + {"dataset", "list", "-o", "json"}, + } { + t.Run(strings.Join(args[:2], " "), func(t *testing.T) { + r := requireSuccess(t, run(t, args...)) + trimmed := strings.TrimSpace(r.Stdout) + require.True(t, strings.HasPrefix(trimmed, "["), + "a list must be a bare array, not an envelope; got:\n%s", firstLine(trimmed)) + + var out []any + r.JSON(t, &out) + }) + } +} + +// TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. +// +// The service answers with a long JSON body. Printing it verbatim buries the +// one useful sentence, so the CLI shortens it, and a regression here is the +// kind that only shows up in someone's terminal. +func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "evaluator", "show", "azdcli-does-not-exist-9999")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) +} + +// TestCLIInitNeedsAnAzdProject covers the whole of what `init` can be asked +// through this harness. +// +// init resolves the project over azd's gRPC channel, so it only works when azd +// is hosting the extension. Running the binary directly there is no host, and +// that is exactly the case a user hits by running the command outside a +// project — so what is asserted is the refusal: it must name `azd init` rather +// than surface a transport error. The scaffolding itself is covered by the +// unit tests, which can supply a fake azd client. +func TestCLIInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + r := requireFailure(t, runIn(t, dir, "init", + "--target", "probe-agent", + "--judge-model", "gpt-4o-mini", + "--no-prompt")) + + require.NotContains(t, r.Combined(), "unknown flag", + "the probe must use init's real flags, or it asserts nothing about projects") + require.Contains(t, r.Combined(), "azd init", + "the refusal must name the command that makes a project") + require.NotContains(t, strings.ToLower(r.Combined()), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, + "a refused init must leave nothing behind") +} + +// TestCLINoPromptFailsInsteadOfHanging is what makes the CLI usable in CI: a +// missing required value must end the process, not wait on a terminal nobody +// is watching. +func TestCLINoPromptFailsInsteadOfHanging(t *testing.T) { + dir := t.TempDir() + r := requireFailure(t, runIn(t, dir, "init", "--no-prompt")) + require.NotEmpty(t, strings.TrimSpace(r.Combined()), + "--no-prompt must say what it could not resolve") +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go new file mode 100644 index 00000000000..178f95e2418 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -0,0 +1,415 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +// The command tests need an eval that has already been run, and building one +// through the CLI is not possible: there is no command that creates an eval +// from flags, only `run start`, which needs a config file and a deployed +// target. So the fixture is built with the client and every assertion is made +// against the binary. What is under test is the command surface; the eval is +// scenery. +// +// It is built once for the whole package because two completed runs cost +// minutes, and torn down in TestMain rather than t.Cleanup so that whichever +// test happened to trigger the build does not take the fixture away from the +// rest. +// +// It evaluates an agent with a built-in evaluator, because that is all M1 can +// run: a deterministic code grader over a target-less dataset would score the +// rows predictably, but code evaluators and no-target runs are both M2. The +// cost is that pass and fail are decided by a judge, so no test may assert how +// many rows failed — only that filtering by verdict is self-consistent. + +const fixtureAPIVersion = "2025-11-15-preview" + +const defaultFixtureModel = "gpt-4o-mini" + +// fixtureQueries are answered by the agent under evaluation. They are ordinary +// support questions: the fixture proves the command surface, not the agent. +var fixtureQueries = []string{ + "How do I reset my password?", + "How do I change my billing address?", + "What are your support hours?", +} + +// evalFixture is one eval with two completed runs. +type evalFixture struct { + EvaluatorName string + EvalID string + + // The agent the runs evaluate, so that a test needing a further run does + // not have to resolve one again. + AgentName string + + // Two runs of the same eval, so that listing, limiting and defaulting to + // the most recent all have something to distinguish. + FirstRunID string + SecondRunID string +} + +var ( + fixtureOnce sync.Once + fixture *evalFixture + fixtureErr error + + // teardown runs after the last test, in reverse order. + teardownMu sync.Mutex + teardown []func() +) + +func deferTeardown(fn func()) { + teardownMu.Lock() + defer teardownMu.Unlock() + teardown = append(teardown, fn) +} + +func runTeardown() { + teardownMu.Lock() + defer teardownMu.Unlock() + for i := len(teardown) - 1; i >= 0; i-- { + teardown[i]() + } + teardown = nil +} + +// runQuietly invokes the binary without a *testing.T. +// +// Teardown runs after the last test has reported, and logging or asserting +// against a finished test panics, so nothing here may touch one. +func runQuietly(args ...string) { + full := append(append([]string{}, args...), "--project-endpoint", endpoint) + _ = exec.Command(binaryPath, full...).Run() +} + +var ( + credOnce sync.Once + cred *azidentity.AzureDeveloperCLICredential + credErr error +) + +// liveClient builds the client the fixture is assembled with. One credential +// for the package, because azidentity caches tokens per instance and a fresh +// one per call makes every call shell out to azd again. +// +// The first token is fetched here rather than lazily on the first request: +// that call is the one that flakes, and paying for it up front means the rest +// of the fixture runs against a cached token. +func liveClient() (*eval_api.EvalClient, error) { + credOnce.Do(func() { + cred, credErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + if credErr != nil { + return + } + credErr = retryCredentialFlake(func() error { + _, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + return err + }) + }) + if credErr != nil { + return nil, credErr + } + return eval_api.NewEvalClient(endpoint, cred), nil +} + +// retryCredentialFlake reruns a request that failed only because azd's token +// helper exited non-zero. +// +// It is the same failure the harness retries around the binary, for the same +// reason: nothing was sent, and the alternative is a suite that fails on a +// different test each run for a reason unrelated to the code. Any other error +// is returned immediately. +func retryCredentialFlake(fn func() error) error { + var err error + for attempt := range 4 { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + if err = fn(); err == nil || !strings.Contains(err.Error(), credentialFlake) { + return err + } + } + return err +} + +// sharedEval returns the fixture, building it on first use. +// +// A failure here fails the calling test rather than skipping it: every test +// that asks for the fixture is testing something that cannot be exercised +// without one, and a suite that goes green because its subject was missing is +// worse than one that goes red. +func sharedEval(t *testing.T) *evalFixture { + t.Helper() + fixtureOnce.Do(func() { + start := time.Now() + fixture, fixtureErr = buildFixture(t.Logf) + t.Logf("fixture ready in %s", time.Since(start).Round(time.Second)) + }) + if fixtureErr != nil { + t.Fatalf("building the shared eval the command tests run against: %v", fixtureErr) + } + return fixture +} + +func fixtureModel() string { + if model := os.Getenv("AZURE_AI_EVAL_MODEL"); model != "" { + return model + } + return defaultFixtureModel +} + +// resolveFixtureAgent names the agent the fixture evaluates. +// +// It reads /agents, not /assistants: they are different collections, and an +// eval target resolves against the former. Naming an assistant is accepted by +// the create and then fails the run with "resources not found". +func resolveFixtureAgent(ctx context.Context) (string, error) { + if name := os.Getenv("AZURE_AI_EVAL_AGENT"); name != "" { + return name, nil + } + + // Builds the shared credential if it does not exist yet; the token below + // comes from it. + if _, err := liveClient(); err != nil { + return "", err + } + + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + if err != nil { + return "", fmt.Errorf("acquiring a token to list agents: %w", err) + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, endpoint+"/agents?api-version="+fixtureAPIVersion, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("listing the project's agents: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf( + "listing the project's agents returned %d; set AZURE_AI_EVAL_AGENT to name one", + resp.StatusCode) + } + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&listing); err != nil { + return "", err + } + for _, a := range listing.Data { + if a.Name != "" { + return a.Name, nil + } + } + return "", fmt.Errorf( + "this project has no agent in /agents, so an agent-target run cannot be built; " + + "deploy an agent or set AZURE_AI_EVAL_AGENT") +} + +func buildFixture(logf func(string, ...any)) (*evalFixture, error) { + ctx := context.Background() + + client, err := liveClient() + if err != nil { + return nil, fmt.Errorf("acquiring an azd credential: %w", err) + } + + agent, err := resolveFixtureAgent(ctx) + if err != nil { + return nil, err + } + logf("evaluating agent %q", agent) + + evaluatorName := "builtin.task_adherence" + evalID, err := createFixtureEval(ctx, client, evaluatorName) + if err != nil { + return nil, err + } + logf("created eval %s", evalID) + + first, err := startFixtureRun(ctx, client, evalID, agent, "first") + if err != nil { + return nil, err + } + second, err := startFixtureRun(ctx, client, evalID, agent, "second") + if err != nil { + return nil, err + } + logf("started runs %s and %s", first, second) + + // Polled together: they are independent, and serializing them doubles the + // slowest part of the suite for nothing. + errs := make(chan error, 2) + for _, runID := range []string{first, second} { + go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) + } + for range 2 { + if err := <-errs; err != nil { + return nil, err + } + } + + return &evalFixture{ + // The criterion is named without the builtin. prefix, and that is the + // name results are reported under. + EvaluatorName: strings.TrimPrefix(evaluatorName, "builtin."), + EvalID: evalID, + AgentName: agent, + FirstRunID: first, + SecondRunID: second, + }, nil +} + +func createFixtureEval( + ctx context.Context, + client *eval_api.EvalClient, + evaluatorName string, +) (string, error) { + criterionName := strings.TrimPrefix(evaluatorName, "builtin.") + + var group *eval_api.OpenAIEval + if err := retryCredentialFlake(func() error { + var err error + group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azdcli-fixture"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: criterionName, + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": fixtureModel(), + "deployment_name": fixtureModel(), + }, + }}, + }) + return err + }); err != nil { + return "", fmt.Errorf("creating the fixture eval: %w", err) + } + deferTeardown(func() { + _ = client.DeleteOpenAIEval(context.Background(), group.ID) + }) + return group.ID, nil +} + +func startFixtureRun( + ctx context.Context, + client *eval_api.EvalClient, + evalID, agentName, label string, +) (string, error) { + rows := make([]map[string]any, 0, len(fixtureQueries)) + for _, q := range fixtureQueries { + rows = append(rows, map[string]any{"query": q}) + } + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent(rows) + + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("azdcli-" + label), + DataSource: ds, + }) + return err + }); err != nil { + return "", fmt.Errorf("starting the %s run: %w", label, err) + } + return run.ID, nil +} + +var terminalRunStatus = map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, +} + +// awaitCompleted requires the run to have scored something. +// +// A run whose every sample errors still reports completed, so the status alone +// would let the whole suite run against an eval that measured nothing. +func awaitCompleted( + ctx context.Context, + client *eval_api.EvalClient, + evalID, runID string, + logf func(string, ...any), +) error { + deadline := time.Now().Add(15 * time.Minute) + for { + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.GetOpenAIEvalRun(ctx, evalID, runID) + return err + }); err != nil { + return fmt.Errorf("polling run %s: %w", runID, err) + } + if terminalRunStatus[strings.ToLower(run.Status)] { + if strings.ToLower(run.Status) != "completed" { + return fmt.Errorf("run %s finished as %q: %s", runID, run.Status, run.Failure()) + } + if run.ResultCounts == nil { + return fmt.Errorf("run %s completed without reporting counts", runID) + } + if run.ResultCounts.Passed+run.ResultCounts.Failed == 0 { + return fmt.Errorf( + "run %s completed without scoring any row (errored=%d); the fixture "+ + "would prove nothing", runID, run.ResultCounts.Errored) + } + logf("run %s completed: passed=%d failed=%d errored=%d", + runID, run.ResultCounts.Passed, run.ResultCounts.Failed, run.ResultCounts.Errored) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("run %s did not finish in time (last status %q)", runID, run.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go new file mode 100644 index 00000000000..804041bde9a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// Generation submits a job that costs model time and takes minutes, so what is +// exercised here is everything up to that point: the flag combinations each +// command refuses and the spec it parses. No test here submits a job. +// +// There is one command per artifact, so nothing suppresses anything: a caller +// who already has a dataset simply does not run `dataset generate`. + +// TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost +// nothing to make. Each is decided locally, so a user finds out before a job is +// billed. +func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { + dir := t.TempDir() + instruction := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(instruction, []byte("test refunds"), 0o600)) + + cases := []struct { + name string + args []string + want string + }{{ + name: "the two instruction sources are mutually exclusive", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--agent-instruction", "inline", "--agent-instruction-file", instruction}, + want: "agent-instruction-file", + }, { + name: "below the minimum sample size", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--max-samples", "14"}, + want: "between 15 and 1000", + }, { + name: "above the maximum sample size", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--max-samples", "1001"}, + want: "between 15 and 1000", + }, { + name: "a missing instruction file names the flag", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--agent-instruction-file", filepath.Join(dir, "absent.md")}, + want: "--agent-instruction-file", + }, { + name: "generating a dataset needs a model deployment", + args: []string{"generate", "--dataset", "--dataset-name", "d", + "--agent-instruction", "inline"}, + want: "--generation-model", + }, { + name: "generating an evaluator needs a model deployment", + args: []string{"generate", "--evaluator", "--evaluator-name", "e", + "--agent-instruction", "inline"}, + want: "--generation-model", + }} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := requireFailure(t, runIn(t, dir, tc.args...)) + require.Contains(t, r.Combined(), tc.want) + }) + } +} + +// TestCLIGenerateNamesTheArtifact pins where the name comes from. The composite +// takes no positional, so with neither a name nor a target there is nothing to +// call the artifact, and the refusal has to name both ways out. +func TestCLIGenerateNamesTheArtifact(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--dataset")) + + require.Contains(t, r.Combined(), "--dataset-name") + require.Contains(t, r.Combined(), "--target") +} + +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with nothing to +// prompt with, the process has to end saying which flag to pass. +// +// The target is no longer among them — it is read from the eval's declaration — +// but the generation model has no other source, so it is the one input a bare +// directory cannot supply. +// +// The target names an agent that does not exist, and that is load-bearing: a +// real one supplies a deployment, and then nothing is missing to report. +func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), + "generate", "--dataset", "--dataset-name", "d", "--target", "a", "--no-prompt")) + require.Contains(t, r.Combined(), "--generation-model") +} + +// TestCLIGenerateDatasetFlagsDoNotApplyToTheEvaluator asserts the scoping the +// help promises is real. +// +// One command now carries both artifacts' settings, so cobra can no longer +// refuse --max-samples on a rubric. What must still hold is that it is not +// *validated* against a generation that does not use it: narrowing to the +// evaluator has to get past the sample-size check, and fail on the model it +// genuinely lacks instead. +func TestCLIGenerateDatasetFlagsDoNotApplyToTheEvaluator(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--evaluator", + "--evaluator-name", "e", "--target", "a", "--max-samples", "20")) + + require.NotContains(t, r.Combined(), "between 15 and 1000", + "--max-samples shapes the dataset, so it must not be validated for a rubric") + require.Contains(t, r.Combined(), "--generation-model", + "it should get as far as the input it actually lacks") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go new file mode 100644 index 00000000000..6751ddd8bde --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// The CI path: start a run without waiting, read the handoff, come back for +// the result later. Everything here is what a pipeline does, so it is driven +// through the binary exactly the way a pipeline would. + +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCLIStartNoWaitEmitsTheHandoff pins the JSON a pipeline reads. +// +// A script captures the run id here and reattaches to it in a later step, so +// the field names are a contract. Emitting the service's run object instead +// would make that script depend on a shape this extension does not control. +func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, + "run", "start", "--eval", f.EvalID, "--no-wait", "-o", "json")) + + var handoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + } + r.JSON(t, &handoff) + + require.NotEmpty(t, handoff.RunID, "a pipeline has nothing to reattach to without run_id") + assert.Equal(t, f.EvalID, handoff.EvalID) + assert.NotEmpty(t, handoff.Status) + + // Started, not finished: this is the whole point of --no-wait, and a + // command that quietly blocked would pass every other assertion here. + assert.NotEqual(t, "completed", handoff.Status) + + deferTeardown(func() { + runQuietly("run", "cancel", handoff.RunID, "--eval", f.EvalID) + }) + + // The id it handed back has to be one the next step can use. + shown := requireSuccess(t, run(t, + "run", "show", handoff.RunID, "--eval", f.EvalID, "-o", "json")) + var reattached struct { + ID string `json:"id"` + } + shown.JSON(t, &reattached) + assert.Equal(t, handoff.RunID, reattached.ID, + "the run id in the handoff must be the one `run show` resolves") +} + +// TestCLIStartNoWaitTellsAPersonHowToReattach covers the same path without +// -o json, where what matters is that the printed command is one that works +// rather than a sentence containing a placeholder. +func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "start", "--eval", f.EvalID, "--no-wait")) + + assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") + assert.Contains(t, r.Stdout, f.EvalID, + "the reattach line must carry the eval id, not a placeholder for it") + assert.NotContains(t, r.Stdout, "<", + "nothing printed for a person to copy may contain a placeholder") + + var runID string + for _, field := range strings.Fields(r.Stdout) { + if strings.HasPrefix(field, "evalrun_") { + runID = field + break + } + } + require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval", f.EvalID) }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go new file mode 100644 index 00000000000..1557ea4aa0d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package cli drives the built binary as a subprocess. +// +// The other live tests call the client layer directly, which proves the API +// paths work but says nothing about the command surface on top of them: flag +// parsing, mutual exclusion, prompting, --no-prompt, exit codes, the rendered +// tables, and whether -o json emits what a script can actually consume. Those +// are the parts a user touches, and until now nothing exercised them against a +// real service. +// +// go test -tags live -v ./tests/cli/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + binaryPath string + endpoint string +) + +// TestMain builds the extension once so every test runs the same binary a user +// would, rather than an in-process command tree that skips main's wiring. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + fmt.Fprintln(os.Stderr, "set AZURE_AI_EVAL_E2E_LIVE=1 to run the CLI tests") + os.Exit(0) + } + + endpoint = strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + fmt.Fprintln(os.Stderr, "FOUNDRY_PROJECT_ENDPOINT is required") + os.Exit(1) + } + + dir, err := os.MkdirTemp("", "azdeval-cli") + if err != nil { + fmt.Fprintf(os.Stderr, "creating a temp dir: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(dir) + + binaryPath = filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binaryPath, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building the extension: %v\n%s\n", err, out) + os.Exit(1) + } + + code := m.Run() + // The shared eval outlives any single test, so it cannot be released with + // t.Cleanup without taking it away from the tests that run after. + runTeardown() + os.RemoveAll(dir) + os.Exit(code) +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// result is one invocation of the binary. +type result struct { + Args []string + Stdout string + Stderr string + ExitCode int +} + +// Combined is stdout and stderr together, for assertions that do not care +// which stream carried the message. +func (r result) Combined() string { return r.Stdout + r.Stderr } + +// JSON decodes stdout, failing the test when the command did not emit +// something a script could consume. +func (r result) JSON(t *testing.T, into any) { + t.Helper() + require.NoError(t, json.Unmarshal([]byte(r.Stdout), into), + "-o json must emit parseable JSON on stdout; got:\n%s", r.Stdout) +} + +// run invokes the binary with the project endpoint already supplied. +func run(t *testing.T, args ...string) result { + t.Helper() + return runIn(t, "", args...) +} + +// credentialFlake is azd's token helper failing under rapid sequential calls. +// +// Every invocation here is a fresh process, so each one shells out to azd for +// a token, and azd intermittently exits non-zero doing it. Retrying is safe +// because no request was made, and the alternative is a suite that fails on a +// different test each run for a reason that has nothing to do with the code. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// runIn invokes the binary with a working directory, for commands that write +// files. +func runIn(t *testing.T, dir string, args ...string) result { + t.Helper() + + res := invoke(t, dir, args...) + for attempt := 0; attempt < 2 && strings.Contains(res.Combined(), credentialFlake); attempt++ { + t.Logf("azd credential flaked; retrying `%s`", strings.Join(args, " ")) + time.Sleep(2 * time.Second) + res = invoke(t, dir, args...) + } + require.NotContains(t, res.Combined(), credentialFlake, + "azd could not produce a token after retries; run `azd auth login` and try again") + return res +} + +func invoke(t *testing.T, dir string, args ...string) result { + t.Helper() + + full := append([]string{}, args...) + if !hasFlag(args, "--project-endpoint") && needsEndpoint(args) { + full = append(full, "--project-endpoint", endpoint) + } + + cmd := exec.Command(binaryPath, full...) + if dir != "" { + cmd.Dir = dir + } + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + code := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("could not run %v: %v", full, err) + } + + res := result{Args: full, Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: code} + t.Logf("$ azd ai eval %s -> exit %d", strings.Join(args, " "), res.ExitCode) + return res +} + +func hasFlag(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// needsEndpoint keeps --project-endpoint off the commands that reject it. +func needsEndpoint(args []string) bool { + for _, a := range args { + switch a { + case "init", "--help", "-h": + return false + } + } + return true +} + +// requireSuccess fails with the command's own output, which is what a user +// would have seen. +func requireSuccess(t *testing.T, r result) result { + t.Helper() + require.Equalf(t, 0, r.ExitCode, + "expected `%s` to succeed\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +// requireFailure asserts a non-zero exit, so a command that silently succeeds +// where it should refuse is caught. +func requireFailure(t *testing.T, r result) result { + t.Helper() + require.NotEqualf(t, 0, r.ExitCode, + "expected `%s` to fail\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go new file mode 100644 index 00000000000..755bab67ffc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// A rubric is the other kind of evaluator: a JSON file of weighted dimensions, +// graded by a judge model rather than by code. It shares nothing with the code +// path on the wire beyond the route, so publishing one had never been +// exercised against a real project. + +// writeRubric lays down a rubric file and returns its path. +func writeRubric(t *testing.T, dimensions string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, + []byte(`{"dimensions":`+dimensions+`}`), 0o600)) + return path +} + +// evaluatorDocument is what `evaluator show` prints. +type evaluatorDocument struct { + Name string `json:"name"` + Version string `json:"version"` + EvaluatorType string `json:"evaluator_type"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Description string `json:"description"` + Weight int `json:"weight"` + } `json:"dimensions"` + DataSchema map[string]any `json:"data_schema"` + InitParameters map[string]any `json:"init_parameters"` + Metrics map[string]any `json:"metrics"` + } `json:"definition"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels"` +} + +// TestCLIRubricRoundTrip publishes a rubric, reads it back, and republishes it. +func TestCLIRubricRoundTrip(t *testing.T) { + name := uniqueName("azdcli_rubric") + rubric := writeRubric(t, `[ + {"id":"tone","description":"Is the answer polite?","weight":5}, + {"id":"accuracy","description":"Is the answer correct?","weight":10} + ]`) + + created := requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", rubric)) + require.Contains(t, created.Stdout, "version 1") + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", name, "-o", "json")) + var doc evaluatorDocument + shown.JSON(t, &doc) + + require.Equal(t, name, doc.Name) + require.Equal(t, "1", doc.Version) + require.Equal(t, "custom", doc.EvaluatorType) + require.Equal(t, "rubric", doc.Definition.Type, + "the discriminator is what tells the service which definition kind it holds") + + require.Len(t, doc.Definition.Dimensions, 2) + byID := map[string]int{} + for _, d := range doc.Definition.Dimensions { + byID[d.ID] = d.Weight + require.NotEmpty(t, d.Description, "a dimension's description is what the judge grades against") + } + require.Equal(t, 5, byID["tone"]) + require.Equal(t, 10, byID["accuracy"]) + + // The rubric named only dimensions. Everything else is filled in by the + // service, and a caller reading the definition back gets those defaults + // rather than what was sent — including the judge model the evaluator will + // require at run time. + require.NotEmpty(t, doc.Definition.DataSchema, + "the service supplies a rubric's data schema; the author never writes one") + require.NotEmpty(t, doc.Definition.InitParameters) + require.NotEmpty(t, doc.Definition.Metrics) + require.NotEmpty(t, doc.SupportedEvaluationLevels) + + // Every registration publishes a new immutable version, which is what + // `update` means for an evaluator. + republished := requireSuccess(t, run(t, "evaluator", "update", name, "--from-file", rubric)) + require.Contains(t, republished.Stdout, "version 2", + "updating must advance the version rather than overwrite") + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "2") + }) + + // The earlier version stays reachable, which is what makes a published + // version safe to reference from a config. + pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1", "-o", "json")) + var first evaluatorDocument + pinned.JSON(t, &first) + require.Equal(t, "1", first.Version) +} + +// TestCLIRubricWeightMustBeAnIntegerFromOneToTen covers the validation a +// hand-authored rubric is most likely to trip. +// +// The service runs two separate checks and they answer differently: a +// fractional weight is rejected for not being an integer, an out-of-range one +// for being out of range. Both are asserted because a caller only ever sees +// one of them, and both have to say what a legal weight is. +func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { + cases := []struct { + name string + weight string + }{ + {"fractional", "2.5"}, + {"zero", "0"}, + {"above ten", "11"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rubric := writeRubric(t, + `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) + + r := requireFailure(t, run(t, "evaluator", "create", + uniqueName("azdcli_badweight"), "--from-file", rubric)) + require.Contains(t, r.Combined(), "between 1 and 10", + "the refusal must say what a legal weight is") + }) + } + + // A weight the service accepts, so the cases above are failing on the + // weight rather than on the rubric shape they share. + name := uniqueName("azdcli_goodweight") + ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", ok)) + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) +} + +// TestCLIRubricNeedsDimensions covers the local check, which costs nothing and +// names the field the service would not. +func TestCLIRubricNeedsDimensions(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + uniqueName("azdcli_nodims"), "--from-file", path)) + require.Contains(t, r.Combined(), "dimensions") +} + +// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show -o json` +// emits JSON a script can consume, whatever the definition kind. It renders the +// service's body rather than a typed struct, so nothing else pins that it stays +// parseable. The bare command renders the human detail view instead. +func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { + name := uniqueName("azdcli_rubricdoc") + + // The wrapped form: a whole evaluator document rather than a bare + // definition. Both are accepted, and generated rubrics arrive wrapped. + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte( + `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, + ), 0o600)) + + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", path)) + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", name, "-o", "json")) + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), + "evaluator show must emit parseable JSON:\n%s", shown.Stdout) + + // The flag names the evaluator, so a name inside the file must not win. + require.Equal(t, name, raw["name"], + "--name must decide the evaluator's name, not the document's own field") + require.NotContains(t, strings.ToLower(shown.Stdout), `"name": "ignored"`) + + // Reconciliation tells people to adopt a remote change by writing it over + // the local definition with --output-file, so the flag has to exist and has + // to land the same document the service holds. + adopted := filepath.Join(t.TempDir(), "adopted.json") + requireSuccess(t, run(t, "evaluator", "show", name, "--output-file", adopted)) + + body, err := os.ReadFile(adopted) + require.NoError(t, err) + var written map[string]any + require.NoError(t, json.Unmarshal(body, &written), + "--output-file must write parseable JSON:\n%s", string(body)) + require.Equal(t, name, written["name"]) + require.Contains(t, written, "definition", + "adopting a remote change needs the definition, not just its identity") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go new file mode 100644 index 00000000000..7b2783a0ef0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type runSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ResultCounts *struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` +} + +func TestCLIRunList(t *testing.T) { + f := sharedEval(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID)) + // These are the columns the spec's `run list` sample prints, in the + // spec's own wording. The old RUN ID/NAME/RESULTS set predates it. + for _, header := range []string{ + "RUN", "DATASET", "STARTED", "STATUS", "SAMPLES", "PASS RATE", + } { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, f.SecondRunID) + require.Regexp(t, `\d+\.\d+%`, r.Stdout, + "the listing must summarise each run's pass rate, not just its status") + }) + + t.Run("json", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "-o", "json")) + require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), + "a list must be a bare array, not the service's envelope") + + var runs []runSummary + r.JSON(t, &runs) + require.GreaterOrEqual(t, len(runs), 2) + + byID := map[string]runSummary{} + for _, entry := range runs { + byID[entry.ID] = entry + } + first, ok := byID[f.FirstRunID] + require.True(t, ok, "the eval's own run is missing from its listing") + require.Equal(t, "completed", first.Status) + require.NotNil(t, first.ResultCounts) + require.Equal(t, len(fixtureQueries), + first.ResultCounts.Passed+first.ResultCounts.Failed, + "every dataset row must be accounted for by a verdict") + }) + + // The client has always taken a limit; until recently the command did not + // expose one, so a service-side truncation would have passed unnoticed. + t.Run("limit", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) + var runs []runSummary + r.JSON(t, &runs) + require.Len(t, runs, 1, "--limit must reach the service") + }) + + t.Run("unknown eval is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "list", "--eval", "eval_azdcli_no_such_eval")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") + }) +} + +func TestCLIRunShow(t *testing.T) { + f := sharedEval(t) + + t.Run("by run id", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval", f.EvalID)) + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "status") + require.Contains(t, r.Stdout, "completed") + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout) + require.Contains(t, r.Stdout, "report") + }) + + // Without --run-id the command has to pick one, and outside an azd + // environment there is no remembered id to fall back on, so what is + // exercised is the listing path. + t.Run("defaults to the most recent run", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) + var newest []runSummary + listed.JSON(t, &newest) + require.Len(t, newest, 1) + + r := requireSuccess(t, run(t, "run", "show", "--eval", f.EvalID, "-o", "json")) + var shown runSummary + r.JSON(t, &shown) + require.Equal(t, newest[0].ID, shown.ID, + "the default must be the run the listing puts first") + }) + + // A remembered run that no longer resolves falls through to the eval's + // latest, but one named explicitly must not: silently showing a different + // run than the one asked for is worse than saying it is gone. + // + // Only the substitution is asserted. Unlike `run list` and `run delete`, + // this path does not shorten the service's body, so the message runs to + // about 1700 characters of raw JSON — recorded in the report rather than + // pinned here, since pinning it would make the length a requirement. + t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope", + "the failure must name the run that was asked for") + require.NotContains(t, r.Combined(), f.FirstRunID, + "an explicit --run-id must not fall back to another run") + }) +} + +// TestCLIRunCancelAndDelete covers both halves of cancel, and the delete that +// follows it, against a single in-flight run: each run costs a minute of +// service time, so the two happy paths share one. +// +// The service answers a cancel on a finished run with success, so without the +// guard the command would tell a user it had stopped something it had not. +func TestCLIRunCancelAndDelete(t *testing.T) { + f := sharedEval(t) + + t.Run("a finished run is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "already finished") + require.Contains(t, r.Combined(), "completed") + }) + + // Delete is covered as far as the service honours it. + // + // The removal itself is not asserted, because whether it happens is the + // service's to decide and it has changed under this test once already: it + // used to accept the DELETE and leave the run readable minutes later, and it + // now reaps a cancelled run promptly enough that the DELETE can even answer + // 404. What is asserted is that the command reaches the right resource — a + // real run is accepted, an unknown one is refused — which is the part that + // would break if the route or the id handling regressed. + t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { + runID := startCancellableRun(t, f) + + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval", f.EvalID)) + require.Contains(t, cancelled.Stdout, runID) + require.Contains(t, cancelled.Stdout, "is now") + + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) + var after runSummary + shown.JSON(t, &after) + require.NotEqual(t, "completed", after.Status, + "a cancelled run must not go on to complete") + + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval", f.EvalID)) + require.Contains(t, deleted.Stdout, "Deleted run") + require.Contains(t, deleted.Stdout, runID) + + // Either outcome is the service's prerogative; what matters is that the + // answer is about this run and not a failure of some other kind. + still := run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json") + if still.ExitCode == 0 { + var survivor runSummary + still.JSON(t, &survivor) + t.Logf("still readable after delete (status %q); the service accepted "+ + "the request without removing anything", survivor.Status) + } else { + require.Contains(t, still.Combined(), runID, + "a read of a deleted run must still name the run it could not find") + t.Log("gone after delete; the service removed it") + } + }) + + // Deleting is not undoable, so the id is required rather than defaulted to + // whichever run happens to be newest. + t.Run("delete requires the run id", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + + t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope") + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + }) +} + +// startCancellableRun adds a run to the fixture's eval and returns it before it +// can finish. +// +// An agent-target run invokes the agent once per row and is judged after that, +// which takes far longer than the second it takes to issue the cancel; a run +// that finished first would turn the cancel test into an assertion about the +// guard it is not testing. +func startCancellableRun(t *testing.T, f *evalFixture) string { + t.Helper() + + client, err := liveClient() + require.NoError(t, err) + + runID, err := startFixtureRun(context.Background(), client, f.EvalID, f.AgentName, "cancelme") + require.NoError(t, err, "starting a run to cancel") + t.Cleanup(func() { + _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) + }) + return runID +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go new file mode 100644 index 00000000000..ab8c07fc96b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/csv" + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// The fixture is judged by a model, so no test here may assert how many rows +// passed. What is under test is the command, and the properties that hold +// whatever the judge decided: every dataset row comes back, every row carries a +// verdict and a score, and filtering by verdict returns a subset that agrees +// with the totals. + +// resultsPayload is what `results show -o json` emits: the run and the rows. +type resultsPayload struct { + Run struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` + PerTestingCriteria []struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"per_testing_criteria_results"` + } `json:"run"` + OutputItems []struct { + ID string `json:"id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item"` + Results []struct { + Name string `json:"name"` + Score *float64 `json:"score"` + Passed bool `json:"passed"` + } `json:"results"` + } `json:"output_items"` +} + +// TestCLIResultsShowRendersTheRows is the difference between `results show` and +// `run show`: the totals say how many failed, these say which. +func TestCLIResultsShowRendersTheRows(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval", f.EvalID)) + + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "Totals:") + require.Contains(t, r.Stdout, "CRITERION") + require.Contains(t, r.Stdout, f.EvaluatorName) + + // One row per evaluated sample, which is what makes "how many should I go + // and look at" answerable by counting lines. + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.NotContains(t, r.Stdout, "EVALUATOR ", + "a per-verdict table would list a sample once per evaluator") + + // The fixture's rows all pass, so every row names no failing evaluator. + require.Contains(t, r.Stdout, "Report:") +} + +func TestCLIResultsShowJSON(t *testing.T) { + f := sharedEval(t) + + payload := resultsFor(t, f.EvalID, f.FirstRunID) + + require.Equal(t, f.FirstRunID, payload.Run.ID) + require.Equal(t, "completed", payload.Run.Status) + require.Equal(t, len(fixtureQueries), payload.Run.ResultCounts.Total) + require.Zero(t, payload.Run.ResultCounts.Errored, + "an errored row means the fixture measured nothing") + + require.Len(t, payload.Run.PerTestingCriteria, 1) + require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) + + // The rows are the reason this command exists, and a run reporting counts + // while returning none would still satisfy everything above. + require.Len(t, payload.OutputItems, len(fixtureQueries), + "every dataset row must come back as an item") + + passed := 0 + for _, item := range payload.OutputItems { + require.NotEmpty(t, item.DataSourceItem["query"], + "each row must carry the column it was evaluated on") + require.Len(t, item.Results, 1) + require.Equal(t, f.EvaluatorName, item.Results[0].Name) + require.NotNil(t, item.Results[0].Score, "a scored row must report its score") + if item.Results[0].Passed { + passed++ + } + } + require.Equal(t, payload.Run.ResultCounts.Passed, passed, + "the per-row verdicts must agree with the totals") +} + +// TestCLIResultsShowFailedOnly asserts the filter removes rows rather than +// merely relabelling them. +// +// The service has no verdict filter — its `status` selects on execution status, +// so `status=failed` returns errored rows, not failing ones — which makes this +// entirely the CLI's own work and worth testing directly. +func TestCLIResultsShowFailedOnly(t *testing.T) { + f := sharedEval(t) + + payload := resultsFor(t, f.EvalID, f.FirstRunID) + + // One rendered row is one evaluator's verdict on one sample, so the count + // to expect is failing *results*, not failing rows: a sample that fails two + // evaluators is two lines. `ResultCounts.Failed` answers the other question. + failing := 0 + for _, item := range payload.OutputItems { + for _, r := range item.Results { + if !r.Passed { + failing++ + } + } + } + + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, + "--eval", f.EvalID, "--failed-only")) + + if failing == 0 { + // Saying so is not the same as printing an empty table. + require.Contains(t, r.Stdout, "No failing rows.") + return + } + + require.NotContains(t, r.Stdout, " pass ", + "--failed-only must drop the rows that passed") + + // Matched on a word boundary so the per-criterion table's FAILED column + // header is not counted as a verdict. + verdicts := regexp.MustCompile(`\bFAIL\b`).FindAllString(r.Stdout, -1) + require.Equal(t, failing, len(verdicts), + "every failing verdict must appear exactly once:\n%s", r.Stdout) +} + +// resultsFor reads a run's results as JSON, which several tests need before +// they can decide what the rendered output should say. +func resultsFor(t *testing.T, evalID, runID string) resultsPayload { + t.Helper() + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval", evalID, "-o", "json")) + var payload resultsPayload + r.JSON(t, &payload) + return payload +} + +func TestCLIResultsExport(t *testing.T) { + f := sharedEval(t) + + t.Run("json to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "json")) + + var exported struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"result_counts"` + } + r.JSON(t, &exported) + require.Equal(t, f.FirstRunID, exported.ID) + require.Equal(t, "completed", exported.Status) + require.Equal(t, len(fixtureQueries), exported.ResultCounts.Total) + }) + + t.Run("csv to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "csv")) + + rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() + require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) + require.Len(t, rows, 2, "a header and one row per criterion") + require.Equal(t, + []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) + require.Equal(t, f.FirstRunID, rows[1][0]) + require.Equal(t, "completed", rows[1][1]) + require.Equal(t, f.EvaluatorName, rows[1][2]) + }) + + t.Run("output-file writes the path instead of stdout", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.csv") + + r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "csv", "--output-file", path)) + require.Empty(t, strings.TrimSpace(r.Stdout), + "--output-file redirects the payload; leaving it on stdout too would double it") + + body, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(body), "run_id,status,criterion,passed,failed") + require.Contains(t, string(body), f.FirstRunID) + }) + + t.Run("an unknown format is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "xml")) + require.Contains(t, r.Combined(), `--format "xml" is not supported`) + // The refusal has to name jsonl too, or it repeats the bug where the + // guard advertised a narrower set than the exporter can write. + require.Contains(t, r.Combined(), "use csv, json or jsonl") + }) + + t.Run("jsonl is accepted, not just advertised", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "results.jsonl") + requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "jsonl", "--output-file", path)) + + body, err := os.ReadFile(path) + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(body)), "\n") + require.NotEmpty(t, lines) + // Every line has to stand alone as an object, otherwise it is JSON + // wearing a .jsonl name. + for _, line := range lines { + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &row)) + } + }) +} + +func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "run", "output", "list", "--eval", "eval_does_not_exist")) + require.Contains(t, r.Combined(), "eval_does_not_exist") + require.NotContains(t, r.Combined(), "RESPONSE 404") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go new file mode 100644 index 00000000000..1ab00592ce1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -0,0 +1,388 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build hero + +// Package hero drives the hero scenarios through real azd, with the extension +// installed the way a user installs it. +// +// The CLI suite in ../cli runs the extension binary directly, which covers the +// command surface but cannot reach `init`: `init` resolves the project and +// edits azure.yaml over azd's gRPC channel, so without azd hosting the process +// there is nothing on the other end. That is not a detail — it is the first +// command in Scenario 1 and the one that produces the local diff every later +// step depends on, and until now the only thing asserting its output was a +// unit test calling the scaffold function directly. A unit test cannot see the +// service entry azd writes, the detection that reads the project, or the +// terminal output the spec pins line for line. +// +// azd x pack --rebuild +// azd extension install azure.ai.evaluations --source local +// go test -tags hero -v ./tests/hero/... +package hero + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// reinstall is what to run when the installed extension is not this code. +// +// `azd x pack` rewrites the artifacts but leaves the checksum in the local +// registry alone when the version has not changed, so a plain reinstall then +// fails validation. Bumping the version in extension.yaml is the way through. +const reinstall = " azd x pack --rebuild\n" + + " azd extension uninstall azure.ai.evaluations\n" + + " azd extension install azure.ai.evaluations --source local\n" + +// TestMain refuses to run against an azd that cannot reach the extension, or +// that is hosting a different build of it. +// +// Skipping would be worse than failing here: these tests exist because nothing +// else covers the azd-hosted path, so a silent skip returns the suite to the +// state it was in before they were written. Running against a stale install is +// worse still — it reports on code that is not the code under test, which is +// the one outcome a test must never produce. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_HERO") != "1" { + fmt.Fprintf(os.Stderr, + "set AZURE_AI_EVAL_HERO=1 to run the hero scenarios. They need azd "+ + "hosting this extension:\n%s", reinstall) + os.Exit(0) + } + + hosted, err := exec.Command("azd", "ai", "eval", "init", "--help").CombinedOutput() + if err != nil || !strings.Contains(string(hosted), "Scaffold evaluation config") { + fmt.Fprintf(os.Stderr, + "azd cannot reach the evaluations extension. Install it first:\n%s\n%s\n", + reinstall, hosted) + os.Exit(1) + } + + if err := requireCurrentInstall(string(hosted)); err != nil { + fmt.Fprintf(os.Stderr, "%v\n\n%s", err, reinstall) + os.Exit(1) + } + + os.Exit(m.Run()) +} + +// requireCurrentInstall compares the installed extension's help against this +// working tree's, so a stale install fails loudly instead of quietly reporting +// on the wrong binary. +// +// Help text is the cheapest available fingerprint that actually moves: it +// carries every command and flag, which is what these tests assert on, and it +// costs one build rather than a version stamp nobody remembers to bump. +func requireCurrentInstall(hosted string) error { + dir, err := os.MkdirTemp("", "azdeval-hero") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + binary := filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binary, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + return fmt.Errorf("building this working tree to compare against: %v\n%s", err, out) + } + + local, err := exec.Command(binary, "init", "--help").CombinedOutput() + if err != nil { + return fmt.Errorf("reading this working tree's help: %w", err) + } + + if normalize(string(local)) != normalize(hosted) { + return fmt.Errorf( + "azd is hosting a different build of this extension.\n"+ + "installed:\n%s\nthis working tree:\n%s", + normalize(hosted), normalize(string(local))) + } + return nil +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// project writes a minimal azd project for `init` to attach to. +// +// It declares the two services detection reads — the Foundry project and the +// agent — because what `init` writes into azure.yaml depends on which of them +// exist, and a project with neither would exercise only the fallback. +func project(t *testing.T, agent string) string { + t.Helper() + dir := t.TempDir() + body := fmt.Sprintf(`name: support-app +services: + ai-project: + host: azure.ai.project + %s: + host: azure.ai.agent +`, agent) + require.NoError(t, os.WriteFile(filepath.Join(dir, "azure.yaml"), []byte(body), 0o600)) + return dir +} + +// azdEval runs the extension through azd, in dir. +func azdEval(t *testing.T, dir string, args ...string) (string, int) { + t.Helper() + + cmd := exec.Command("azd", append([]string{"ai", "eval"}, args...)...) + cmd.Dir = dir + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + code := 0 + if err := cmd.Run(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("could not run azd ai eval %v: %v", args, err) + } + code = exitErr.ExitCode() + } + + // azd prints its own upgrade notice to stderr, which is not the command's + // output and would break an exact comparison. + text := dropUpgradeNotice(out.String()) + t.Logf("$ azd ai eval %s -> exit %d\n%s", strings.Join(args, " "), code, text) + return text, code +} + +// dropUpgradeNotice removes azd's "Update available" banner and everything +// after it, which azd appends regardless of the command. +func dropUpgradeNotice(s string) string { + if i := strings.Index(s, "Update available:"); i >= 0 { + s = s[:i] + } + return strings.TrimRight(s, " \r\n\t") +} + +// normalize makes terminal output comparable across platforms. +func normalize(s string) string { + return strings.ReplaceAll(dropUpgradeNotice(s), "\r\n", "\n") +} + +// TestHeroScenario1ColdStart is the first half of Scenario 1: the offline +// baseline `init` writes, asserted against the terminal block the spec shows. +// +// The output is compared whole rather than by keyword. Every line of it is a +// promise the spec makes to a reader deciding whether to adopt this — which +// files appear, what was detected, what to run next — and a keyword assertion +// would pass while the reader's terminal said something else. +func TestHeroScenario1ColdStart(t *testing.T) { + const ( + agent = "support-agent" + judge = "gpt-5.6-luna" + ) + dir := project(t, agent) + + // The evaluator and judge are passed rather than prompted for, because the + // spec's two `?` lines are answers to prompts and a test has no terminal to + // answer them at. + out, code := azdEval(t, dir, "init", + "--target", agent, "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", judge) + require.Zero(t, code, "init makes no service calls, so nothing can fail it here") + + want := `(✓) Done: Detected agent target: support-agent +(✓) Done: Using data source: traces (Application Insights) +(✓) Done: Judge model deployment: gpt-5.6-luna + +Created + evals/azure.eval.yaml evaluation configuration + azure.yaml added service 'support-agent-evals' + +Next: azd up + azd ai eval run start` + + require.Equal(t, want, normalize(out)) +} + +// Scenario 1's second half: the azure.eval.yaml the terminal block promised. The spec +// prints this file, so its shape is as much a promise as the output above — +// and it is the file a reader reviews before running `azd up`. +func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", + "--target", "support-agent", "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", "gpt-5.6-luna") + require.Zero(t, code) + + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) + require.NoError(t, err) + text := string(body) + + require.Contains(t, text, "name: support-agent-trace-eval") + require.Contains(t, text, "type: traces") + require.Contains(t, text, "agent_name: support-agent", + "a trace run has no target, so agent_name is what scopes it") + require.Contains(t, text, "max_traces: 20", + "a first run is bounded rather than taking the service default of 1000") + require.Contains(t, text, "evaluator: builtin.task_adherence") + require.Contains(t, text, "model: gpt-5.6-luna", + "the judge is written per evaluator reference as initialization_parameters.model") + + require.NotContains(t, text, "datasets:", + "there is no file to register, so the catalog is absent rather than empty") + require.NotContains(t, text, "target:", + "a trace run invokes nothing") +} + +// `init` is offline, and being offline is the property that makes its output a +// reviewable local diff. A service call here would also make the command fail +// for a user who has not authenticated yet, which is exactly when they run it. +func TestHeroInitMakesNoServiceCalls(t *testing.T) { + dir := project(t, "support-agent") + + cmd := exec.Command("azd", "ai", "eval", "init", + "--target", "support-agent", "--evaluator", "builtin.task_adherence", + "--judge-model", "m") + cmd.Dir = dir + // A proxy pointing nowhere fails any outbound request, so a command that + // stays offline is unaffected and one that does not cannot be mistaken for + // working. + cmd.Env = append(os.Environ(), + "HTTPS_PROXY=http://127.0.0.1:9", + "HTTP_PROXY=http://127.0.0.1:9", + "NO_PROXY=", + ) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, "init must not need the network:\n%s", out) +} + +// The eval service has to be declared in azure.yaml before azd will act on it. +// Printing the block and leaving the edit to the reader was enough, once, to +// make the documented flow stop working between `init` and `azd up`. +func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m") + require.Zero(t, code) + + root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + text := string(root) + + require.Contains(t, text, "support-agent-evals:", + "the service is named for the agent it evaluates") + require.Contains(t, text, "host: azure.ai.eval") + require.Contains(t, text, "$ref: ./evals/azure.eval.yaml") + + // azd owns the edit, so everything the project already declared survives it. + require.Contains(t, text, "name: support-app") + require.Contains(t, text, "host: azure.ai.project") + require.Contains(t, text, "host: azure.ai.agent") + + // The eval reads both, so azd has to deploy both first. + require.Regexp(t, `(?s)support-agent-evals:.*uses:.*ai-project.*support-agent`, text) +} + +// Running `init` twice must not deploy the same eval twice. The service key is +// the eval's name, so the second run recognizes its own work. +func TestHeroInitIsIdempotent(t *testing.T) { + dir := project(t, "support-agent") + args := []string{"init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m"} + + _, code := azdEval(t, dir, args...) + require.Zero(t, code) + first, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + + out, code := azdEval(t, dir, args...) + require.NotZero(t, code, "the scaffold already exists, so a second run must refuse") + require.Contains(t, out, "--force", "the refusal has to say how to proceed") + + second, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, string(first), string(second), + "a refused init must not have edited the project") + + // With --force the files are rewritten, and the service is still declared + // exactly once. + out, code = azdEval(t, dir, append(args, "--force")...) + require.Zero(t, code, out) + + third, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), + "a second eval service would deploy the same eval twice") + require.Contains(t, normalize(out), "already declares service 'support-agent-evals'") +} + +// Evals attach to a project; they do not create one. Naming the command that +// makes a project is more use than a transport error from the gRPC channel +// that was not there. +func TestHeroInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + out, code := azdEval(t, dir, "init", "--target", "support-agent", "--no-prompt") + require.NotZero(t, code) + require.Contains(t, out, "azd init") + require.NotContains(t, strings.ToLower(out), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a refused init must leave nothing behind") +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation — so the "next" steps must stop offering to generate one. +func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--judge-model", "m", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.NotContains(t, text, "evaluator generate", + "nothing was scheduled to be generated, so nothing should be suggested") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "evaluator: builtin.task_adherence") + require.NotContains(t, string(body), "support-agent-quality", + "the default rubric was replaced, not added to") +} + +// A supplied dataset is not generated either, so `init` has nothing left to +// suggest and must not send the reader to a command that would submit a job +// for an artifact they already have. +func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--judge-model", "m", + "--dataset", "prod-golden", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.NotContains(t, text, "dataset generate") + require.Contains(t, text, "Next: azd up", + "with nothing left to generate, the next step is the deploy") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "dataset: prod-golden") + require.NotContains(t, string(body), "source:", + "a registered dataset has nothing to upload") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go new file mode 100644 index 00000000000..32dda623b17 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -0,0 +1,422 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package live holds integration tests that talk to a real Foundry project. +// They are excluded from the default build by the `live` tag and additionally +// gated on AZURE_AI_EVAL_E2E_LIVE so an accidental run cannot create resources. +// +// go test -tags live -v ./tests/live/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +// +// Optional: +// +// AZURE_AI_EVAL_MODEL= (default gpt-4.1-nano) +// AZURE_AI_EVAL_AGENT= (enables the run phase) +package live + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +const ( + projectAPIVersion = "2025-11-15-preview" + defaultJudgeModel = "gpt-4.1-nano" + sampleDatasetContent = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +{"query":"Can I change my shipping address after ordering?"} +` +) + +type liveEnv struct { + endpoint string + judgeModel string + agentName string + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + // Works non-interactively when azd already holds a refresh token, + // which is what makes an unattended run possible. + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + }) + return sharedCred, sharedCredErr +} + +func setup(t *testing.T) *liveEnv { + t.Helper() + + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + + cred, err := liveCredential() + require.NoError(t, err, "acquiring an azd credential") + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = defaultJudgeModel + } + + return &liveEnv{ + endpoint: endpoint, + judgeModel: judge, + agentName: os.Getenv("AZURE_AI_EVAL_AGENT"), + evalClient: eval_api.NewEvalClient(endpoint, cred), + datasetClient: dataset_api.NewDatasetClient(endpoint, cred), + } +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().Unix()) +} + +// pickQualityEvaluator selects a built-in whose required inputs match the +// agent-target data mapping this extension sends. +// +// Built-ins do not share one input contract: builtin.ifeval, for example, +// requires an `instruction_id_list` field, and creating a group with it under +// the agent-target mapping fails with MissingRequiredDataMapping. The +// agent-target mapping supplies query, response, tool_calls and +// tool_definitions, so the evaluators below are the compatible set. +func pickQualityEvaluator(t *testing.T, available []eval_api.EvaluatorSummary) string { + t.Helper() + + preferred := []string{ + "builtin.task_adherence", + "builtin.task_completion", + "builtin.tool_call_accuracy", + } + present := map[string]bool{} + for _, e := range available { + present[e.Name] = true + } + for _, name := range preferred { + if present[name] { + return name + } + } + + names := make([]string, 0, len(available)) + for _, e := range available { + names = append(names, e.Name) + } + t.Skipf("no agent-target compatible evaluator found; available: %s", strings.Join(names, ", ")) + return "" +} + +// TestLiveBuiltinEvaluators is the cheapest reachability check: it proves the +// endpoint, credential, api-version, and auth scope are all correct without +// creating anything. +func TestLiveBuiltinEvaluators(t *testing.T) { + env := setup(t) + ctx := context.Background() + + list, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion, + ) + require.NoError(t, err, "listing built-in evaluators") + require.NotEmpty(t, list.Value, "the project should expose built-in evaluators") + + t.Logf("found %d built-in evaluators; first: %s", len(list.Value), list.Value[0].Name) +} + +// TestLiveDatasetLifecycle exercises the full pending-upload flow and asserts +// that re-registering the same name yields the next version rather than an error. +func TestLiveDatasetLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + dir := t.TempDir() + require.NoError(t, + os.WriteFile(filepath.Join(dir, "golden.jsonl"), []byte(sampleDatasetContent), 0o600)) + + name := uniqueName("azd-eval-e2e") + + first, err := env.datasetClient.UploadNewVersion(ctx, name, "", dir, projectAPIVersion) + require.NoError(t, err, "registering the first dataset version") + require.Equal(t, name, first.Name) + require.NotEmpty(t, first.Version) + t.Logf("registered %s version %s", first.Name, first.Version) + + t.Cleanup(func() { + // Best effort: leave nothing behind even if the test fails midway. + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, first.Version, projectAPIVersion) + }) + + fetched, err := env.datasetClient.GetDataset(ctx, name, first.Version, projectAPIVersion) + require.NoError(t, err, "reading the dataset back") + t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) + + // The version listing is eventually consistent: it returns nothing for a + // second or two after a version is created, even though the version itself + // reads back fine. Poll rather than asserting on the first response. + var versions *dataset_api.DatasetList + require.Eventually(t, func() bool { + var err error + versions, err = env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + return err == nil && versions != nil && len(versions.Value) > 0 + }, 30*time.Second, 2*time.Second, "the version listing never caught up") + require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) + + // A second upload must advance the version, not conflict. + second, err := env.datasetClient.UploadNewVersion( + ctx, name, first.Version, dir, projectAPIVersion) + require.NoError(t, err, "registering a second dataset version") + require.NotEqual(t, first.Version, second.Version, + "re-registering the same name must produce the next version") + t.Cleanup(func() { + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, second.Version, projectAPIVersion) + }) +} + +// TestLiveEvalLifecycle proves the create request this extension builds is +// accepted, which is the single most important contract to get right. +func TestLiveEvalLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + threshold := 3.0 + req := &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-group"), + Metadata: map[string]string{"azd_source": "e2e"}, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + "threshold": threshold, + }, + }}, + } + + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + require.NoError(t, err, "creating the eval") + require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") + t.Logf("created eval %s (name %q)", group.ID, group.Name) + + fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) + require.NoError(t, err, "reading the eval back") + require.Equal(t, group.ID, fetched.ID) +} + +// resolveAgent names the agent the run phase evaluates. +// +// AZURE_AI_EVAL_AGENT wins when set. Otherwise one is discovered, and failing +// to find one is a failure rather than a skip: skipping by default is how the +// agent-target path went unverified for weeks while the suite reported green. +// +// The listing is /agents, not /assistants. They are different collections and +// a project can have plenty of the latter and none of the former — an eval +// target resolves against /agents, so an assistant name is accepted by the +// request and then fails the run with "resources not found". +func resolveAgent(t *testing.T, env *liveEnv) string { + t.Helper() + + if env.agentName != "" { + return env.agentName + } + + cred, err := liveCredential() + require.NoError(t, err) + token, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + require.NoError(t, err, "acquiring a token to list agents") + + req, err := http.NewRequest(http.MethodGet, env.endpoint+"/agents?api-version="+projectAPIVersion, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err, "listing the project's agents") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, + "could not list agents; set AZURE_AI_EVAL_AGENT to name one directly") + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&listing)) + + for _, a := range listing.Data { + if a.Name != "" { + t.Logf("no AZURE_AI_EVAL_AGENT set; evaluating %q", a.Name) + return a.Name + } + } + + t.Fatal("this project has no agent in /agents, so the agent-target run path " + + "cannot be verified here. Assistants do not count: an eval target " + + "resolves against /agents, and naming an assistant fails the run with " + + "\"resources not found\". Deploy an agent, or set AZURE_AI_EVAL_AGENT " + + "to one in another project") + return "" +} + +// TestLiveRun invokes a real agent, which is the only cover the agent-target +// run path has. +func TestLiveRun(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-run"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval for the run") + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + + // A single sample is roughly 40 seconds; allow generous headroom. + deadline := time.Now().Add(10 * time.Minute) + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + t.Logf("run reached %s", current.Status) + if current.ResultCounts != nil { + t.Logf("counts: passed=%d failed=%d errored=%d", + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + + // Reaching a terminal state is not the same as having evaluated + // anything. A run whose every sample errors still reports + // "completed", so asserting only on the status would let the target + // or the evaluator break without the test noticing. + require.Equal(t, "completed", strings.ToLower(current.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, current.ResultCounts, "a completed run must report counts") + require.Zero(t, current.ResultCounts.Errored, + "an errored sample means the target or the evaluator did not run") + require.Positive(t, + current.ResultCounts.Passed+current.ResultCounts.Failed, + "the run must score at least one sample; a pass or a fail are both fine, "+ + "but scoring nothing means the data never reached the evaluator") + return + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within the deadline (last status %q)", + run.ID, current.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go new file mode 100644 index 00000000000..6b53228f03f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveRunCancel covers the one route whose meaning depends on the request +// body. POST on the run cancels it when the body is empty and updates its +// status and counters when it is not, so a stray body here would silently +// overwrite a run instead of stopping it. Only a live call can tell the two +// apart: both are the same method on the same path, and both return 200. +func TestLiveRunCancel(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-cancel"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval to cancel a run from") + + t.Cleanup(func() { + _ = env.evalClient.DeleteOpenAIEval(context.Background(), group.ID) + }) + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("cancel"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run to cancel") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + canceled, err := env.evalClient.CancelOpenAIEvalRun(ctx, group.ID, run.ID) + if err != nil { + // The service refuses to cancel a run that already left the cancellable + // window. That is a race this test starts but does not control, and it + // is the behaviour a separate case already pins, so there is nothing + // left here to observe. + current, getErr := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, getErr, "reading the run whose cancel was refused") + + // Only that race is skipped. A refusal while the run is still moving is + // the failure this test exists to catch, and skipping on any error at + // all retires the test without anyone deciding to. + require.Truef(t, runAlreadyFinished(current.Status), + "cancel was refused while the run was still %q: %v", current.Status, err) + t.Skipf("cancel was refused with the run already at %q: %v", current.Status, err) + } + require.NotNil(t, canceled) + t.Logf("cancel returned status %s", canceled.Status) + + // A sample takes roughly 40 seconds, so a run cancelled immediately after + // it starts should never reach completed. + deadline := time.Now().Add(5 * time.Minute) + var status string + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the cancelled run") + status = strings.ToLower(current.Status) + if status == "canceled" || status == "cancelled" { + break + } + require.NotEqual(t, "completed", status, + "the run completed instead of cancelling, so the empty-body POST did not cancel it") + require.False(t, time.Now().After(deadline), + "the run never reached a cancelled state; last status was %s", status) + time.Sleep(5 * time.Second) + } + + t.Logf("run reached %s", status) +} + +// runAlreadyFinished reports a run that has stopped, so a refused cancel is the +// race rather than a fault. Spelled out here because tests/live cannot reach +// internal/cmd's copy, and both spellings of cancelled are in use. +func runAlreadyFinished(status string) bool { + switch strings.ToLower(status) { + case "completed", "failed", "canceled", "cancelled", "error": + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt new file mode 100644 index 00000000000..f8730a057a6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -0,0 +1 @@ +1.0.14-beta diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..329c4a2181e --- /dev/null +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -0,0 +1,45 @@ +# Continuous deployment trigger +trigger: + branches: + include: + - main + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - /eng/pipelines/templates/stages/release-azd-extension.yml + - /eng/pipelines/templates/jobs/build-azd-extension.yml + - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml + - /eng/pipelines/templates/variables/image.yml + +pr: + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-ext-azure-ai-evaluations.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml + - eng/pipelines/templates/steps/publish-cli.yml + exclude: + - cli/azd/docs/** + +parameters: + - name: PublishToRegistry + displayName: Publish to registry + type: string + # Scheduled (nightly) runs override this in the shared templates; the runtime + # parameter default must be a literal because it renders before variables exist. + default: stable + values: + - stable + - dev + - nightly + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - template: /eng/pipelines/templates/stages/release-azd-extension.yml + parameters: + AzdExtensionId: azure.ai.evaluations + SanitizedExtensionId: azure-ai-evaluations + AzdExtensionDirectory: cli/azd/extensions/azure.ai.evaluations + PublishToRegistry: ${{ parameters.PublishToRegistry }}